1use std::collections::{HashMap, HashSet};
16use std::fmt::Display;
17use std::sync::Arc;
18
19use serde::{Deserialize, Deserializer, Serialize};
20use snafu::{OptionExt, ResultExt, ensure};
21use store_api::storage::{MAX_REGION_SEQ, RegionId, RegionNumber};
22use table::metadata::TableId;
23
24use crate::error::{
25 InvalidMetadataSnafu, MetadataCorruptionSnafu, RegionNotFoundSnafu, Result, SerdeJsonSnafu,
26 TableRouteNotFoundSnafu, UnexpectedLogicalRouteTableSnafu,
27};
28use crate::key::node_address::{NodeAddressKey, NodeAddressValue};
29use crate::key::txn_helper::TxnOpGetResponseSet;
30use crate::key::{
31 DeserializedValueWithBytes, MetadataKey, MetadataValue, RegionDistribution,
32 TABLE_ROUTE_KEY_PATTERN, TABLE_ROUTE_PREFIX,
33};
34use crate::kv_backend::KvBackendRef;
35use crate::kv_backend::txn::Txn;
36use crate::rpc::router::{RegionRoute, region_distribution};
37use crate::rpc::store::BatchGetRequest;
38
39#[derive(Debug, PartialEq)]
43pub struct TableRouteKey {
44 pub table_id: TableId,
45}
46
47impl TableRouteKey {
48 pub fn new(table_id: TableId) -> Self {
49 Self { table_id }
50 }
51
52 pub fn range_prefix() -> Vec<u8> {
54 format!("{}/", TABLE_ROUTE_PREFIX).into_bytes()
55 }
56}
57
58#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
59#[serde(tag = "type", rename_all = "snake_case")]
60pub enum TableRouteValue {
61 Physical(PhysicalTableRouteValue),
62 Logical(LogicalTableRouteValue),
63}
64
65#[derive(Clone)]
67pub enum TableRoute {
68 Physical(Arc<PhysicalTableRouteValue>),
69 Logical(Arc<LogicalTableRouteValue>),
70}
71
72impl From<TableRouteValue> for TableRoute {
73 fn from(value: TableRouteValue) -> Self {
74 match value {
75 TableRouteValue::Physical(route) => Self::Physical(Arc::new(route)),
76 TableRouteValue::Logical(route) => Self::Logical(Arc::new(route)),
77 }
78 }
79}
80
81impl From<Arc<TableRoute>> for TableRoute {
82 fn from(route: Arc<TableRoute>) -> Self {
83 Arc::unwrap_or_clone(route)
84 }
85}
86
87impl TableRoute {
88 pub fn is_physical(&self) -> bool {
90 matches!(self, TableRoute::Physical(_))
91 }
92
93 pub fn as_physical_table_route_ref(&self) -> Option<&Arc<PhysicalTableRouteValue>> {
95 match self {
96 TableRoute::Physical(table_route) => Some(table_route),
97 TableRoute::Logical(_) => None,
98 }
99 }
100}
101
102#[derive(Debug, PartialEq, Serialize, Clone, Default)]
103pub struct PhysicalTableRouteValue {
104 pub region_routes: Vec<RegionRoute>,
106 pub max_region_number: RegionNumber,
110 version: u64,
112}
113
114impl<'de> Deserialize<'de> for PhysicalTableRouteValue {
115 fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
116 where
117 D: Deserializer<'de>,
118 {
119 #[derive(Deserialize)]
120 struct Helper {
121 region_routes: Vec<RegionRoute>,
122 #[serde(default)]
123 max_region_number: Option<RegionNumber>,
124 version: u64,
125 }
126
127 let mut helper = Helper::deserialize(deserializer)?;
128 if helper.max_region_number.is_none() {
130 let max_region = helper
131 .region_routes
132 .iter()
133 .map(|r| r.region.id.region_number())
134 .max()
135 .unwrap_or_default();
136 helper.max_region_number = Some(max_region);
137 }
138
139 Ok(PhysicalTableRouteValue {
140 region_routes: helper.region_routes,
141 max_region_number: helper.max_region_number.unwrap_or_default(),
142 version: helper.version,
143 })
144 }
145}
146
147#[derive(Debug, PartialEq, Serialize, Deserialize, Clone)]
148pub struct LogicalTableRouteValue {
149 physical_table_id: TableId,
150}
151
152impl TableRouteValue {
153 pub(crate) fn new(
156 table_id: TableId,
157 physical_table_id: TableId,
158 region_routes: Vec<RegionRoute>,
159 ) -> Self {
160 if table_id == physical_table_id {
161 TableRouteValue::physical(region_routes)
162 } else {
163 TableRouteValue::logical(physical_table_id)
164 }
165 }
166
167 pub fn physical(region_routes: Vec<RegionRoute>) -> Self {
168 Self::Physical(PhysicalTableRouteValue::new(region_routes))
169 }
170
171 pub fn logical(physical_table_id: TableId) -> Self {
172 Self::Logical(LogicalTableRouteValue::new(physical_table_id))
173 }
174
175 pub fn update(&self, region_routes: Vec<RegionRoute>) -> Result<Self> {
177 ensure!(
178 self.is_physical(),
179 UnexpectedLogicalRouteTableSnafu {
180 err_msg: format!("{self:?} is a non-physical TableRouteValue."),
181 }
182 );
183 let physical_table_route = self.as_physical_table_route_ref();
184 let original_max_region_number = physical_table_route.max_region_number;
185 let new_max_region_number = region_routes
186 .iter()
187 .map(|r| r.region.id.region_number())
188 .max()
189 .unwrap_or_default();
190 let version = physical_table_route.version;
191 Ok(Self::Physical(PhysicalTableRouteValue {
192 region_routes,
193 max_region_number: original_max_region_number.max(new_max_region_number),
196 version: version + 1,
197 }))
198 }
199
200 #[cfg(any(test, feature = "testing"))]
204 pub fn version(&self) -> Result<u64> {
205 ensure!(
206 self.is_physical(),
207 UnexpectedLogicalRouteTableSnafu {
208 err_msg: format!("{self:?} is a non-physical TableRouteValue."),
209 }
210 );
211 Ok(self.as_physical_table_route_ref().version)
212 }
213
214 pub fn region_route(&self, region_id: RegionId) -> Result<Option<RegionRoute>> {
218 ensure!(
219 self.is_physical(),
220 UnexpectedLogicalRouteTableSnafu {
221 err_msg: format!("{self:?} is a non-physical TableRouteValue."),
222 }
223 );
224 Ok(self
225 .as_physical_table_route_ref()
226 .region_routes
227 .iter()
228 .find(|route| route.region.id == region_id)
229 .cloned())
230 }
231
232 pub fn is_physical(&self) -> bool {
234 matches!(self, TableRouteValue::Physical(_))
235 }
236
237 pub fn region_routes(&self) -> Result<&Vec<RegionRoute>> {
239 ensure!(
240 self.is_physical(),
241 UnexpectedLogicalRouteTableSnafu {
242 err_msg: format!("{self:?} is a non-physical TableRouteValue."),
243 }
244 );
245 Ok(&self.as_physical_table_route_ref().region_routes)
246 }
247
248 pub fn max_region_number(&self) -> Result<RegionNumber> {
253 ensure!(
254 self.is_physical(),
255 UnexpectedLogicalRouteTableSnafu {
256 err_msg: format!("{self:?} is a non-physical TableRouteValue."),
257 }
258 );
259 Ok(self.as_physical_table_route_ref().max_region_number)
260 }
261
262 fn as_physical_table_route_ref(&self) -> &PhysicalTableRouteValue {
267 match self {
268 TableRouteValue::Physical(x) => x,
269 _ => unreachable!("Mistakenly been treated as a Physical TableRoute: {self:?}"),
270 }
271 }
272
273 pub fn into_physical_table_route(self) -> PhysicalTableRouteValue {
278 match self {
279 TableRouteValue::Physical(x) => x,
280 _ => unreachable!("Mistakenly been treated as a Physical TableRoute: {self:?}"),
281 }
282 }
283
284 pub fn into_logical_table_route(self) -> LogicalTableRouteValue {
289 match self {
290 TableRouteValue::Logical(x) => x,
291 _ => unreachable!("Mistakenly been treated as a Logical TableRoute: {self:?}"),
292 }
293 }
294
295 pub fn region_numbers(&self) -> Vec<RegionNumber> {
296 match self {
297 TableRouteValue::Physical(x) => x
298 .region_routes
299 .iter()
300 .map(|region_route| region_route.region.id.region_number())
301 .collect(),
302 TableRouteValue::Logical(_) => {
303 vec![]
304 }
305 }
306 }
307}
308
309impl MetadataValue for TableRouteValue {
310 fn try_from_raw_value(raw_value: &[u8]) -> Result<Self> {
311 let r = serde_json::from_slice::<TableRouteValue>(raw_value);
312 match r {
313 Err(e) if e.is_data() => Ok(Self::Physical(
315 serde_json::from_slice::<PhysicalTableRouteValue>(raw_value)
316 .context(SerdeJsonSnafu)?,
317 )),
318 Ok(x) => Ok(x),
319 Err(e) => Err(e).context(SerdeJsonSnafu),
320 }
321 }
322
323 fn try_as_raw_value(&self) -> Result<Vec<u8>> {
324 serde_json::to_vec(self).context(SerdeJsonSnafu)
325 }
326}
327
328impl PhysicalTableRouteValue {
329 pub fn version(&self) -> u64 {
331 self.version
332 }
333 pub fn new(region_routes: Vec<RegionRoute>) -> Self {
334 let max_region_number = region_routes
335 .iter()
336 .map(|r| r.region.id.region_number())
337 .max()
338 .unwrap_or_default();
339 Self {
340 region_routes,
341 max_region_number,
342 version: 0,
343 }
344 }
345}
346
347impl LogicalTableRouteValue {
348 pub fn new(physical_table_id: TableId) -> Self {
349 Self { physical_table_id }
350 }
351
352 pub fn physical_table_id(&self) -> TableId {
353 self.physical_table_id
354 }
355}
356
357impl MetadataKey<'_, TableRouteKey> for TableRouteKey {
358 fn to_bytes(&self) -> Vec<u8> {
359 self.to_string().into_bytes()
360 }
361
362 fn from_bytes(bytes: &[u8]) -> Result<TableRouteKey> {
363 let key = std::str::from_utf8(bytes).map_err(|e| {
364 InvalidMetadataSnafu {
365 err_msg: format!(
366 "TableRouteKey '{}' is not a valid UTF8 string: {e}",
367 String::from_utf8_lossy(bytes)
368 ),
369 }
370 .build()
371 })?;
372 let captures = TABLE_ROUTE_KEY_PATTERN
373 .captures(key)
374 .context(InvalidMetadataSnafu {
375 err_msg: format!("Invalid TableRouteKey '{key}'"),
376 })?;
377 let table_id = captures[1].parse::<TableId>().unwrap();
379 Ok(TableRouteKey { table_id })
380 }
381}
382
383impl Display for TableRouteKey {
384 fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
385 write!(f, "{}/{}", TABLE_ROUTE_PREFIX, self.table_id)
386 }
387}
388
389pub type TableRouteManagerRef = Arc<TableRouteManager>;
390
391pub struct TableRouteManager {
392 storage: TableRouteStorage,
393}
394
395impl TableRouteManager {
396 pub fn new(kv_backend: KvBackendRef) -> Self {
397 Self {
398 storage: TableRouteStorage::new(kv_backend),
399 }
400 }
401
402 pub async fn get_physical_table_id(
407 &self,
408 logical_or_physical_table_id: TableId,
409 ) -> Result<TableId> {
410 let table_route = self
411 .storage
412 .get_inner(logical_or_physical_table_id)
413 .await?
414 .context(TableRouteNotFoundSnafu {
415 table_id: logical_or_physical_table_id,
416 })?;
417
418 match table_route {
419 TableRouteValue::Physical(_) => Ok(logical_or_physical_table_id),
420 TableRouteValue::Logical(x) => Ok(x.physical_table_id()),
421 }
422 }
423
424 pub async fn get_physical_table_route(
430 &self,
431 logical_or_physical_table_id: TableId,
432 ) -> Result<(TableId, PhysicalTableRouteValue)> {
433 let table_route = self
434 .storage
435 .get(logical_or_physical_table_id)
436 .await?
437 .context(TableRouteNotFoundSnafu {
438 table_id: logical_or_physical_table_id,
439 })?;
440
441 match table_route {
442 TableRouteValue::Physical(x) => Ok((logical_or_physical_table_id, x)),
443 TableRouteValue::Logical(x) => {
444 let physical_table_id = x.physical_table_id();
445 let physical_table_route = self.storage.get(physical_table_id).await?.context(
446 TableRouteNotFoundSnafu {
447 table_id: physical_table_id,
448 },
449 )?;
450 let physical_table_route = physical_table_route.into_physical_table_route();
451 Ok((physical_table_id, physical_table_route))
452 }
453 }
454 }
455
456 pub async fn batch_get_physical_table_routes(
463 &self,
464 logical_or_physical_table_ids: &[TableId],
465 ) -> Result<HashMap<TableId, PhysicalTableRouteValue>> {
466 let routes =
467 Self::resolve_physical_table_routes(logical_or_physical_table_ids, |ids| async move {
468 self.storage.batch_get(&ids).await
469 })
470 .await?;
471 Ok(routes
472 .into_iter()
473 .map(|(id, route)| (id, Arc::unwrap_or_clone(route)))
474 .collect())
475 }
476
477 pub async fn resolve_physical_table_routes<R, Fut>(
480 ids: &[TableId],
481 load: impl Fn(Vec<TableId>) -> Fut,
482 ) -> Result<HashMap<TableId, Arc<PhysicalTableRouteValue>>>
483 where
484 R: Into<TableRoute>,
485 Fut: Future<Output = Result<Vec<Option<R>>>>,
486 {
487 let requested = Self::index_routes(ids, load(ids.to_vec()).await?);
488 let missing_targets = requested
489 .values()
490 .filter_map(|route| match route {
491 TableRoute::Logical(logical)
492 if !requested.contains_key(&logical.physical_table_id()) =>
493 {
494 Some(logical.physical_table_id())
495 }
496 _ => None,
497 })
498 .collect::<HashSet<_>>();
499
500 let targets = if missing_targets.is_empty() {
501 HashMap::new()
502 } else {
503 let ids = missing_targets.into_iter().collect::<Vec<_>>();
504 let routes = load(ids.clone()).await?;
505 Self::index_routes(&ids, routes)
506 };
507
508 let mut resolved = HashMap::with_capacity(requested.len());
509 for (&id, route) in &requested {
510 let physical = match route {
511 TableRoute::Physical(physical) => physical,
512 TableRoute::Logical(logical) => {
513 let table_id = logical.physical_table_id();
514 let target = requested
515 .get(&table_id)
516 .or_else(|| targets.get(&table_id))
517 .context(TableRouteNotFoundSnafu { table_id })?;
518 target.as_physical_table_route_ref().with_context(|| {
519 MetadataCorruptionSnafu {
520 err_msg: format!(
521 "logical table {id} points to a logical route {table_id}."
522 ),
523 }
524 })?
525 }
526 };
527 resolved.insert(id, physical.clone());
528 }
529 Ok(resolved)
530 }
531
532 fn index_routes<R: Into<TableRoute>>(
533 ids: &[TableId],
534 routes: Vec<Option<R>>,
535 ) -> HashMap<TableId, TableRoute> {
536 ids.iter()
537 .copied()
538 .zip(routes)
539 .filter_map(|(id, route)| route.map(|route| (id, route.into())))
540 .collect()
541 }
542
543 pub async fn get_region_distribution(
545 &self,
546 table_id: TableId,
547 ) -> Result<Option<RegionDistribution>> {
548 self.storage
549 .get(table_id)
550 .await?
551 .map(|table_route| Ok(region_distribution(table_route.region_routes()?)))
552 .transpose()
553 }
554
555 pub async fn try_reserve_region_numbers(
559 &self,
560 table_id: TableId,
561 current: &DeserializedValueWithBytes<TableRouteValue>,
562 count: usize,
563 ) -> Result<Option<RegionNumber>> {
564 let TableRouteValue::Physical(mut route) =
566 TableRouteValue::try_from_raw_value(¤t.bytes)?
567 else {
568 return UnexpectedLogicalRouteTableSnafu {
569 err_msg: format!("Cannot reserve region numbers for logical table {table_id}"),
570 }
571 .fail();
572 };
573 let previous = route.max_region_number;
574 let last = RegionNumber::try_from(count)
575 .ok()
576 .and_then(|count| previous.checked_add(count))
577 .filter(|last| previous < *last && *last <= MAX_REGION_SEQ)
578 .with_context(|| InvalidMetadataSnafu {
579 err_msg: format!(
580 "Cannot reserve {count} region numbers after {previous} for table {table_id}"
581 ),
582 })?;
583 route.max_region_number = last;
584 route.version += 1;
585 let (txn, _) =
586 self.storage
587 .build_update_txn(table_id, current, &TableRouteValue::Physical(route))?;
588 let response = self.storage.kv_backend.txn(txn).await?;
589 Ok(response.succeeded.then_some(previous + 1))
590 }
591
592 pub async fn set_region_staging_state(
598 &self,
599 region_id: store_api::storage::RegionId,
600 staging: bool,
601 ) -> Result<()> {
602 let table_id = region_id.table_id();
603
604 let current_table_route = self
606 .storage
607 .get_with_raw_bytes(table_id)
608 .await?
609 .context(TableRouteNotFoundSnafu { table_id })?;
610
611 let new_table_route = current_table_route.inner.clone();
613
614 ensure!(
616 new_table_route.is_physical(),
617 UnexpectedLogicalRouteTableSnafu {
618 err_msg: format!("Cannot set staging state for logical table {table_id}"),
619 }
620 );
621
622 let region_routes = new_table_route.region_routes()?.clone();
623 let mut updated_routes = region_routes.clone();
624
625 let mut region_found = false;
628 for route in &mut updated_routes {
629 if route.region.id == region_id {
630 if staging {
631 route.set_leader_staging();
632 } else {
633 route.clear_leader_staging();
634 }
635 region_found = true;
636 break;
637 }
638 }
639
640 ensure!(region_found, RegionNotFoundSnafu { region_id });
641
642 let updated_table_route = new_table_route.update(updated_routes)?;
644
645 let (txn, _) =
647 self.storage
648 .build_update_txn(table_id, ¤t_table_route, &updated_table_route)?;
649
650 let result = self.storage.kv_backend.txn(txn).await?;
651
652 ensure!(
653 result.succeeded,
654 MetadataCorruptionSnafu {
655 err_msg: format!(
656 "Failed to update staging state for region {}: CAS operation failed",
657 region_id
658 ),
659 }
660 );
661
662 Ok(())
663 }
664
665 pub async fn is_region_staging(&self, region_id: store_api::storage::RegionId) -> Result<bool> {
669 let table_id = region_id.table_id();
670
671 let table_route = self.storage.get(table_id).await?;
672
673 match table_route {
674 Some(route) if route.is_physical() => {
675 let region_routes = route.region_routes()?;
676 for route in region_routes {
677 if route.region.id == region_id {
678 return Ok(route.is_leader_staging());
679 }
680 }
681 Ok(false)
682 }
683 _ => Ok(false),
684 }
685 }
686
687 pub fn table_route_storage(&self) -> &TableRouteStorage {
689 &self.storage
690 }
691}
692
693pub struct TableRouteStorage {
695 kv_backend: KvBackendRef,
696}
697
698pub type TableRouteValueDecodeResult = Result<Option<DeserializedValueWithBytes<TableRouteValue>>>;
699
700impl TableRouteStorage {
701 pub fn new(kv_backend: KvBackendRef) -> Self {
702 Self { kv_backend }
703 }
704
705 pub fn build_create_txn(
708 &self,
709 table_id: TableId,
710 table_route_value: &TableRouteValue,
711 ) -> Result<(
712 Txn,
713 impl FnOnce(&mut TxnOpGetResponseSet) -> TableRouteValueDecodeResult + use<>,
714 )> {
715 let key = TableRouteKey::new(table_id);
716 let raw_key = key.to_bytes();
717
718 let txn = Txn::put_if_not_exists(raw_key.clone(), table_route_value.try_as_raw_value()?);
719
720 Ok((
721 txn,
722 TxnOpGetResponseSet::decode_with(TxnOpGetResponseSet::filter(raw_key)),
723 ))
724 }
725
726 pub fn build_update_txn(
731 &self,
732 table_id: TableId,
733 current_table_route_value: &DeserializedValueWithBytes<TableRouteValue>,
734 new_table_route_value: &TableRouteValue,
735 ) -> Result<(
736 Txn,
737 impl FnOnce(&mut TxnOpGetResponseSet) -> TableRouteValueDecodeResult + use<>,
738 )> {
739 let key = TableRouteKey::new(table_id);
740 let raw_key = key.to_bytes();
741 let raw_value = current_table_route_value.get_raw_bytes();
742 let new_raw_value: Vec<u8> = new_table_route_value.try_as_raw_value()?;
743
744 let txn = Txn::compare_and_put(raw_key.clone(), raw_value, new_raw_value);
745
746 Ok((
747 txn,
748 TxnOpGetResponseSet::decode_with(TxnOpGetResponseSet::filter(raw_key)),
749 ))
750 }
751
752 pub async fn get(&self, table_id: TableId) -> Result<Option<TableRouteValue>> {
754 let mut table_route = self.get_inner(table_id).await?;
755 if let Some(table_route) = &mut table_route {
756 self.remap_table_route(table_route).await?;
757 };
758
759 Ok(table_route)
760 }
761
762 async fn get_inner(&self, table_id: TableId) -> Result<Option<TableRouteValue>> {
763 let key = TableRouteKey::new(table_id);
764 self.kv_backend
765 .get(&key.to_bytes())
766 .await?
767 .map(|kv| TableRouteValue::try_from_raw_value(&kv.value))
768 .transpose()
769 }
770
771 pub async fn get_with_raw_bytes(
773 &self,
774 table_id: TableId,
775 ) -> Result<Option<DeserializedValueWithBytes<TableRouteValue>>> {
776 let mut table_route = self.get_with_raw_bytes_inner(table_id).await?;
777 if let Some(table_route) = &mut table_route {
778 self.remap_table_route(table_route).await?;
779 };
780
781 Ok(table_route)
782 }
783
784 async fn get_with_raw_bytes_inner(
785 &self,
786 table_id: TableId,
787 ) -> Result<Option<DeserializedValueWithBytes<TableRouteValue>>> {
788 let key = TableRouteKey::new(table_id);
789 self.kv_backend
790 .get(&key.to_bytes())
791 .await?
792 .map(|kv| DeserializedValueWithBytes::from_inner_slice(&kv.value))
793 .transpose()
794 }
795
796 pub async fn batch_get(&self, table_ids: &[TableId]) -> Result<Vec<Option<TableRouteValue>>> {
798 let raw_table_routes = self.batch_get_with_raw_bytes(table_ids).await?;
799
800 Ok(raw_table_routes
801 .into_iter()
802 .map(|v| v.map(|x| x.inner))
803 .collect())
804 }
805
806 pub async fn batch_get_with_raw_bytes(
811 &self,
812 table_ids: &[TableId],
813 ) -> Result<Vec<Option<DeserializedValueWithBytes<TableRouteValue>>>> {
814 let mut raw_table_routes = self.batch_get_inner(table_ids).await?;
815 self.remap_routes_addresses(&mut raw_table_routes).await?;
816
817 Ok(raw_table_routes)
818 }
819
820 async fn batch_get_inner(
821 &self,
822 table_ids: &[TableId],
823 ) -> Result<Vec<Option<DeserializedValueWithBytes<TableRouteValue>>>> {
824 let keys = table_ids
825 .iter()
826 .map(|id| TableRouteKey::new(*id).to_bytes())
827 .collect::<Vec<_>>();
828 let resp = self
829 .kv_backend
830 .batch_get(BatchGetRequest { keys: keys.clone() })
831 .await?;
832
833 let kvs = resp
834 .kvs
835 .into_iter()
836 .map(|kv| (kv.key, kv.value))
837 .collect::<HashMap<_, _>>();
838 keys.into_iter()
839 .map(|key| {
840 if let Some(value) = kvs.get(&key) {
841 Ok(Some(DeserializedValueWithBytes::from_inner_slice(value)?))
842 } else {
843 Ok(None)
844 }
845 })
846 .collect()
847 }
848
849 async fn remap_routes_addresses(
850 &self,
851 table_routes: &mut [Option<DeserializedValueWithBytes<TableRouteValue>>],
852 ) -> Result<()> {
853 let keys = table_routes
854 .iter()
855 .flat_map(|table_route| {
856 table_route
857 .as_ref()
858 .map(|x| extract_address_keys(&x.inner))
859 .unwrap_or_default()
860 })
861 .collect::<HashSet<_>>()
862 .into_iter()
863 .collect();
864 let node_addrs = self.get_node_addresses(keys).await?;
865 for table_route in table_routes.iter_mut().flatten() {
866 set_addresses(&node_addrs, table_route)?;
867 }
868
869 Ok(())
870 }
871
872 pub(crate) async fn remap_table_route(&self, table_route: &mut TableRouteValue) -> Result<()> {
873 let keys = extract_address_keys(table_route).into_iter().collect();
874 let node_addrs = self.get_node_addresses(keys).await?;
875 set_addresses(&node_addrs, table_route)?;
876
877 Ok(())
878 }
879
880 pub(crate) async fn remap_region_routes(
881 &self,
882 region_routes: &mut [RegionRoute],
883 ) -> Result<()> {
884 let keys = extract_address_keys_from_region_routes(region_routes)
885 .into_iter()
886 .collect();
887 let node_addrs = self.get_node_addresses(keys).await?;
888 set_addresses_for_region_routes(&node_addrs, region_routes)
889 }
890
891 async fn get_node_addresses(
892 &self,
893 keys: Vec<Vec<u8>>,
894 ) -> Result<HashMap<u64, NodeAddressValue>> {
895 if keys.is_empty() {
896 return Ok(HashMap::default());
897 }
898
899 self.kv_backend
900 .batch_get(BatchGetRequest { keys })
901 .await?
902 .kvs
903 .into_iter()
904 .map(|kv| {
905 let node_id = NodeAddressKey::from_bytes(&kv.key)?.node_id;
906 let node_addr = NodeAddressValue::try_from_raw_value(&kv.value)?;
907 Ok((node_id, node_addr))
908 })
909 .collect()
910 }
911}
912
913fn set_addresses_for_region_routes(
914 node_addrs: &HashMap<u64, NodeAddressValue>,
915 region_routes: &mut [RegionRoute],
916) -> Result<()> {
917 for region_route in region_routes {
918 if let Some(leader) = &mut region_route.leader_peer
919 && let Some(node_addr) = node_addrs.get(&leader.id)
920 {
921 leader.addr = node_addr.peer.addr.clone();
922 }
923 for follower in &mut region_route.follower_peers {
924 if let Some(node_addr) = node_addrs.get(&follower.id) {
925 follower.addr = node_addr.peer.addr.clone();
926 }
927 }
928 }
929
930 Ok(())
931}
932
933fn set_addresses(
934 node_addrs: &HashMap<u64, NodeAddressValue>,
935 table_route: &mut TableRouteValue,
936) -> Result<()> {
937 let TableRouteValue::Physical(physical_table_route) = table_route else {
938 return Ok(());
939 };
940 set_addresses_for_region_routes(node_addrs, &mut physical_table_route.region_routes)
941}
942
943fn extract_address_keys_from_region_routes(region_routes: &[RegionRoute]) -> HashSet<Vec<u8>> {
944 region_routes
945 .iter()
946 .flat_map(|region_route| {
947 region_route
948 .follower_peers
949 .iter()
950 .map(|peer| NodeAddressKey::with_datanode(peer.id).to_bytes())
951 .chain(
952 region_route
953 .leader_peer
954 .as_ref()
955 .map(|leader| NodeAddressKey::with_datanode(leader.id).to_bytes()),
956 )
957 })
958 .collect()
959}
960
961fn extract_address_keys(table_route: &TableRouteValue) -> HashSet<Vec<u8>> {
962 let TableRouteValue::Physical(physical_table_route) = table_route else {
963 return HashSet::default();
964 };
965
966 extract_address_keys_from_region_routes(&physical_table_route.region_routes)
967}
968
969#[cfg(test)]
970mod tests {
971 use std::sync::Arc;
972
973 use super::*;
974 use crate::kv_backend::memory::MemoryKvBackend;
975 use crate::kv_backend::{KvBackend, TxnService};
976 use crate::peer::Peer;
977 use crate::rpc::router::Region;
978 use crate::rpc::store::PutRequest;
979
980 #[test]
981 fn test_update_table_route_max_region_number() {
982 let table_route = PhysicalTableRouteValue::new(vec![
983 RegionRoute {
984 region: Region {
985 id: RegionId::new(0, 1),
986 ..Default::default()
987 },
988 ..Default::default()
989 },
990 RegionRoute {
991 region: Region {
992 id: RegionId::new(0, 2),
993 ..Default::default()
994 },
995 ..Default::default()
996 },
997 ]);
998 assert_eq!(table_route.max_region_number, 2);
999
1000 let new_table_route = TableRouteValue::Physical(table_route)
1002 .update(vec![RegionRoute {
1003 region: Region {
1004 id: RegionId::new(0, 1),
1005 ..Default::default()
1006 },
1007 ..Default::default()
1008 }])
1009 .unwrap();
1010 assert_eq!(
1011 new_table_route
1012 .as_physical_table_route_ref()
1013 .max_region_number,
1014 2
1015 );
1016
1017 let new_table_route = new_table_route
1019 .update(vec![RegionRoute {
1020 region: Region {
1021 id: RegionId::new(0, 3),
1022 ..Default::default()
1023 },
1024 ..Default::default()
1025 }])
1026 .unwrap()
1027 .into_physical_table_route();
1028 assert_eq!(new_table_route.max_region_number, 3);
1029 }
1030
1031 #[tokio::test]
1032 async fn test_reserve_region_numbers() {
1033 let table_id = 1024;
1034 let kv = Arc::new(MemoryKvBackend::default());
1035 let manager = TableRouteManager::new(kv.clone());
1036 let storage = manager.table_route_storage();
1037 let routes = vec![RegionRoute {
1038 region: Region {
1039 id: RegionId::new(table_id, MAX_REGION_SEQ - 3),
1040 ..Default::default()
1041 },
1042 ..Default::default()
1043 }];
1044 let initial = TableRouteValue::physical(routes.clone());
1045 let (txn, _) = storage.build_create_txn(table_id, &initial).unwrap();
1046 assert!(kv.txn(txn).await.unwrap().succeeded);
1047 let current = storage.get_with_raw_bytes(table_id).await.unwrap().unwrap();
1048 assert!(
1049 manager
1050 .try_reserve_region_numbers(table_id, ¤t, 0)
1051 .await
1052 .is_err()
1053 );
1054 assert_eq!(
1055 manager
1056 .try_reserve_region_numbers(table_id, ¤t, 2)
1057 .await
1058 .unwrap(),
1059 Some(MAX_REGION_SEQ - 2)
1060 );
1061 assert_eq!(
1063 manager
1064 .try_reserve_region_numbers(table_id, ¤t, 2)
1065 .await
1066 .unwrap(),
1067 None
1068 );
1069 let current = storage.get_with_raw_bytes(table_id).await.unwrap().unwrap();
1070 assert_eq!(current.region_routes().unwrap(), &routes);
1071 assert_eq!(current.max_region_number().unwrap(), MAX_REGION_SEQ - 1);
1072 assert_eq!(current.version().unwrap(), 1);
1073 assert!(
1074 manager
1075 .try_reserve_region_numbers(table_id, ¤t, 2)
1076 .await
1077 .is_err()
1078 );
1079 assert_eq!(
1080 manager
1081 .try_reserve_region_numbers(table_id, ¤t, 1)
1082 .await
1083 .unwrap(),
1084 Some(MAX_REGION_SEQ)
1085 );
1086 let current = storage.get_with_raw_bytes(table_id).await.unwrap().unwrap();
1087 assert!(
1088 manager
1089 .try_reserve_region_numbers(table_id, ¤t, 1)
1090 .await
1091 .is_err()
1092 );
1093 }
1094
1095 #[test]
1096 fn test_table_route_compatibility() {
1097 let old_raw_v = r#"{"region_routes":[{"region":{"id":1,"name":"r1","partition":null,"attrs":{}},"leader_peer":{"id":2,"addr":"a2"},"follower_peers":[]},{"region":{"id":1,"name":"r1","partition":null,"attrs":{}},"leader_peer":{"id":2,"addr":"a2"},"follower_peers":[]}],"version":0}"#;
1098 let v = TableRouteValue::try_from_raw_value(old_raw_v.as_bytes()).unwrap();
1099
1100 let expected_table_route = TableRouteValue::Physical(PhysicalTableRouteValue {
1101 region_routes: vec![
1102 RegionRoute {
1103 region: Region {
1104 id: RegionId::new(0, 1),
1105 name: "r1".to_string(),
1106 attrs: Default::default(),
1107 partition_expr: Default::default(),
1108 },
1109 leader_peer: Some(Peer {
1110 id: 2,
1111 addr: "a2".to_string(),
1112 }),
1113 follower_peers: vec![],
1114 leader_state: None,
1115 leader_down_since: None,
1116 write_route_policy: None,
1117 },
1118 RegionRoute {
1119 region: Region {
1120 id: RegionId::new(0, 1),
1121 name: "r1".to_string(),
1122 attrs: Default::default(),
1123 partition_expr: Default::default(),
1124 },
1125 leader_peer: Some(Peer {
1126 id: 2,
1127 addr: "a2".to_string(),
1128 }),
1129 follower_peers: vec![],
1130 leader_state: None,
1131 leader_down_since: None,
1132 write_route_policy: None,
1133 },
1134 ],
1135 max_region_number: 1,
1136 version: 0,
1137 });
1138
1139 assert_eq!(v, expected_table_route);
1140 }
1141
1142 #[test]
1143 fn test_key_serialization() {
1144 let key = TableRouteKey::new(42);
1145 let raw_key = key.to_bytes();
1146 assert_eq!(raw_key, b"__table_route/42");
1147 }
1148
1149 #[test]
1150 fn test_key_deserialization() {
1151 let expected = TableRouteKey::new(42);
1152 let key = TableRouteKey::from_bytes(b"__table_route/42").unwrap();
1153 assert_eq!(key, expected);
1154 }
1155
1156 #[tokio::test]
1157 async fn test_table_route_storage_get_with_raw_bytes_empty() {
1158 let kv = Arc::new(MemoryKvBackend::default());
1159 let table_route_storage = TableRouteStorage::new(kv);
1160 let table_route = table_route_storage.get_with_raw_bytes(1024).await.unwrap();
1161 assert!(table_route.is_none());
1162 }
1163
1164 #[tokio::test]
1165 async fn test_table_route_storage_get_with_raw_bytes() {
1166 let kv = Arc::new(MemoryKvBackend::default());
1167 let table_route_storage = TableRouteStorage::new(kv.clone());
1168 let table_route = table_route_storage.get_with_raw_bytes(1024).await.unwrap();
1169 assert!(table_route.is_none());
1170 let table_route_manager = TableRouteManager::new(kv.clone());
1171 let table_route_value = TableRouteValue::Logical(LogicalTableRouteValue {
1172 physical_table_id: 1023,
1173 });
1174 let (txn, _) = table_route_manager
1175 .table_route_storage()
1176 .build_create_txn(1024, &table_route_value)
1177 .unwrap();
1178 let r = kv.txn(txn).await.unwrap();
1179 assert!(r.succeeded);
1180 let table_route = table_route_storage.get_with_raw_bytes(1024).await.unwrap();
1181 assert!(table_route.is_some());
1182 let got = table_route.unwrap().inner;
1183 assert_eq!(got, table_route_value);
1184 }
1185
1186 #[tokio::test]
1187 async fn test_table_route_batch_get() {
1188 let kv = Arc::new(MemoryKvBackend::default());
1189 let table_route_storage = TableRouteStorage::new(kv.clone());
1190 let routes = table_route_storage
1191 .batch_get(&[1023, 1024, 1025])
1192 .await
1193 .unwrap();
1194
1195 assert!(routes.iter().all(Option::is_none));
1196 let table_route_manager = TableRouteManager::new(kv.clone());
1197 let routes = [
1198 (
1199 1024,
1200 TableRouteValue::Logical(LogicalTableRouteValue {
1201 physical_table_id: 1023,
1202 }),
1203 ),
1204 (
1205 1025,
1206 TableRouteValue::Logical(LogicalTableRouteValue {
1207 physical_table_id: 1023,
1208 }),
1209 ),
1210 ];
1211 for (table_id, route) in &routes {
1212 let (txn, _) = table_route_manager
1213 .table_route_storage()
1214 .build_create_txn(*table_id, route)
1215 .unwrap();
1216 let r = kv.txn(txn).await.unwrap();
1217 assert!(r.succeeded);
1218 }
1219
1220 let results = table_route_storage
1221 .batch_get(&[9999, 1025, 8888, 1024])
1222 .await
1223 .unwrap();
1224 assert!(results[0].is_none());
1225 assert_eq!(results[1].as_ref().unwrap(), &routes[1].1);
1226 assert!(results[2].is_none());
1227 assert_eq!(results[3].as_ref().unwrap(), &routes[0].1);
1228 }
1229
1230 #[tokio::test]
1231 async fn test_batch_resolve_logical_routes() {
1232 let routes = HashMap::from([
1233 (1, TableRouteValue::logical(3)),
1234 (2, TableRouteValue::logical(3)),
1235 (3, TableRouteValue::physical(vec![])),
1236 (4, TableRouteValue::logical(5)),
1237 ]);
1238 let load = |ids: Vec<TableId>| {
1239 std::future::ready(Ok(ids.iter().map(|id| routes.get(id).cloned()).collect()))
1240 };
1241 let resolved = TableRouteManager::resolve_physical_table_routes(&[1, 2, 3, 9], load)
1242 .await
1243 .unwrap();
1244 assert_eq!(
1245 resolved.keys().copied().collect::<HashSet<_>>(),
1246 HashSet::from([1, 2, 3])
1247 );
1248 assert!(Arc::ptr_eq(&resolved[&1], &resolved[&3]));
1249 assert!(Arc::ptr_eq(&resolved[&2], &resolved[&3]));
1250 let logical_only = TableRouteManager::resolve_physical_table_routes(&[1, 2], load)
1251 .await
1252 .unwrap();
1253 assert_eq!(logical_only.len(), 2);
1254 assert!(Arc::ptr_eq(&logical_only[&1], &logical_only[&2]));
1255 let missing = TableRouteManager::resolve_physical_table_routes(&[4], load)
1256 .await
1257 .unwrap_err();
1258 assert!(matches!(
1259 missing,
1260 crate::error::Error::TableRouteNotFound { table_id: 5, .. }
1261 ));
1262 }
1263
1264 #[tokio::test]
1265 async fn test_table_route_cache_batches_backend_reads() {
1266 use crate::cache::new_table_route_cache;
1267 use crate::cache::test_util::assert_batch_cache_reads;
1268
1269 assert_batch_cache_reads(new_table_route_cache, |id| {
1270 let value = TableRouteValue::logical(id + 100);
1271 (
1272 TableRouteKey::new(id).to_bytes(),
1273 value.try_as_raw_value().unwrap(),
1274 )
1275 })
1276 .await;
1277 }
1278
1279 #[tokio::test]
1280 async fn test_table_route_batch_get_remaps_addresses() {
1281 let kv = Arc::new(MemoryKvBackend::default());
1282 let table_route_storage = TableRouteStorage::new(kv.clone());
1283 let table_route_manager = TableRouteManager::new(kv.clone());
1284 let table_route_value = TableRouteValue::Physical(PhysicalTableRouteValue {
1285 region_routes: vec![RegionRoute {
1286 leader_peer: Some(Peer {
1287 id: 1,
1288 addr: "old-leader".to_string(),
1289 }),
1290 follower_peers: vec![Peer {
1291 id: 2,
1292 addr: "old-follower".to_string(),
1293 }],
1294 ..Default::default()
1295 }],
1296 max_region_number: 0,
1297 version: 0,
1298 });
1299 let (txn, _) = table_route_manager
1300 .table_route_storage()
1301 .build_create_txn(1024, &table_route_value)
1302 .unwrap();
1303 let r = kv.txn(txn).await.unwrap();
1304 assert!(r.succeeded);
1305
1306 let cache = crate::cache::new_table_route_cache(
1308 "routes".into(),
1309 moka::future::Cache::new(8),
1310 kv.clone(),
1311 );
1312 let cached = cache.get_many(&[1024]).await.unwrap();
1313 let physical = cached[0]
1314 .as_ref()
1315 .unwrap()
1316 .as_physical_table_route_ref()
1317 .unwrap();
1318 assert_eq!(
1319 physical.region_routes[0].leader_peer.as_ref().unwrap().addr,
1320 "old-leader"
1321 );
1322
1323 for (node_id, addr) in [(1, "new-leader"), (2, "new-follower")] {
1324 kv.put(PutRequest {
1325 key: NodeAddressKey::with_datanode(node_id).to_bytes(),
1326 value: NodeAddressValue {
1327 peer: Peer {
1328 id: node_id,
1329 addr: addr.to_string(),
1330 },
1331 }
1332 .try_as_raw_value()
1333 .unwrap(),
1334 ..Default::default()
1335 })
1336 .await
1337 .unwrap();
1338 }
1339
1340 let results = table_route_storage.batch_get(&[1024]).await.unwrap();
1341 let TableRouteValue::Physical(physical_table_route) = results[0].as_ref().unwrap() else {
1342 panic!("Expected PhysicalTableRouteValue");
1343 };
1344
1345 let region_route = &physical_table_route.region_routes[0];
1346 assert_eq!(
1347 region_route.leader_peer.as_ref().unwrap().addr,
1348 "new-leader"
1349 );
1350 assert_eq!(region_route.follower_peers[0].addr, "new-follower");
1351 }
1352
1353 #[tokio::test]
1354 async fn remap_route_address_updates_addresses() {
1355 let kv = Arc::new(MemoryKvBackend::default());
1356 let table_route_storage = TableRouteStorage::new(kv.clone());
1357 let mut table_route = TableRouteValue::Physical(PhysicalTableRouteValue {
1358 region_routes: vec![RegionRoute {
1359 leader_peer: Some(Peer {
1360 id: 1,
1361 ..Default::default()
1362 }),
1363 follower_peers: vec![Peer {
1364 id: 2,
1365 ..Default::default()
1366 }],
1367 ..Default::default()
1368 }],
1369 max_region_number: 0,
1370 version: 0,
1371 });
1372
1373 kv.put(PutRequest {
1374 key: NodeAddressKey::with_datanode(1).to_bytes(),
1375 value: NodeAddressValue {
1376 peer: Peer {
1377 addr: "addr1".to_string(),
1378 ..Default::default()
1379 },
1380 }
1381 .try_as_raw_value()
1382 .unwrap(),
1383 ..Default::default()
1384 })
1385 .await
1386 .unwrap();
1387
1388 table_route_storage
1389 .remap_table_route(&mut table_route)
1390 .await
1391 .unwrap();
1392
1393 if let TableRouteValue::Physical(physical_table_route) = table_route {
1394 assert_eq!(
1395 physical_table_route.region_routes[0]
1396 .leader_peer
1397 .as_ref()
1398 .unwrap()
1399 .addr,
1400 "addr1"
1401 );
1402 assert_eq!(
1403 physical_table_route.region_routes[0].follower_peers[0].addr,
1404 ""
1405 );
1406 } else {
1407 panic!("Expected PhysicalTableRouteValue");
1408 }
1409 }
1410}