1pub(crate) mod datanode;
16pub(crate) mod health;
17pub(crate) mod heartbeat;
18pub(crate) mod leader;
19pub(crate) mod maintenance;
20pub(crate) mod node_lease;
21pub(crate) mod procedure;
22pub(crate) mod recovery;
23pub(crate) mod sequencer;
24mod util;
25
26use std::collections::HashMap;
27use std::convert::Infallible;
28use std::sync::Arc;
29use std::task::{Context, Poll};
30
31use axum::{Router as AxumRouter, routing};
32use tonic::body::Body;
33use tonic::codegen::{BoxFuture, Service, http};
34use tonic::server::NamedService;
35
36use crate::metasrv::Metasrv;
37use crate::service::admin::heartbeat::HeartBeatHandler;
38use crate::service::admin::leader::LeaderHandler;
39use crate::service::admin::maintenance::MaintenanceHandler;
40use crate::service::admin::node_lease::NodeLeaseHandler;
41use crate::service::admin::procedure::ProcedureManagerHandler;
42use crate::service::admin::recovery::RecoveryHandler;
43use crate::service::admin::sequencer::TableIdAllocatorHandler;
44
45pub fn make_admin_service(metasrv: Arc<Metasrv>) -> Admin {
52 let router = Router::new().route("/health", health::HealthHandler);
53
54 let router = router.route(
55 "/node-lease",
56 node_lease::NodeLeaseHandler {
57 meta_peer_client: metasrv.meta_peer_client().clone(),
58 },
59 );
60
61 let handler = heartbeat::HeartBeatHandler {
62 meta_peer_client: metasrv.meta_peer_client().clone(),
63 };
64 let router = router
65 .route("/heartbeat", handler.clone())
66 .route("/heartbeat/help", handler);
67
68 let router = router.route(
69 "/leader",
70 leader::LeaderHandler {
71 election: metasrv.election().cloned(),
72 },
73 );
74
75 let router = router.routes(
76 &[
77 "/maintenance",
78 "/maintenance/status",
79 "/maintenance/enable",
80 "/maintenance/disable",
81 ],
82 maintenance::MaintenanceHandler {
83 manager: metasrv.runtime_switch_manager().clone(),
84 },
85 );
86 let router = router.routes(
87 &[
88 "/procedure-manager/pause",
89 "/procedure-manager/resume",
90 "/procedure-manager/status",
91 ],
92 procedure::ProcedureManagerHandler {
93 manager: metasrv.runtime_switch_manager().clone(),
94 },
95 );
96 let router = Router::nest("/admin", router);
97
98 Admin::new(router)
99}
100
101#[async_trait::async_trait]
102pub trait HttpHandler: Send + Sync {
103 async fn handle(
104 &self,
105 path: &str,
106 method: http::Method,
107 params: &HashMap<String, String>,
108 ) -> crate::Result<http::Response<String>>;
109}
110
111#[derive(Clone)]
112pub struct Admin
113where
114 Self: Send,
115{
116 router: Arc<Router>,
117}
118
119impl Admin {
120 pub fn new(router: Router) -> Self {
121 Self {
122 router: Arc::new(router),
123 }
124 }
125}
126
127impl NamedService for Admin {
128 const NAME: &'static str = "admin";
129}
130
131impl Service<http::Request<Body>> for Admin {
132 type Response = http::Response<Body>;
133 type Error = Infallible;
134 type Future = BoxFuture<Self::Response, Self::Error>;
135
136 fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
137 Poll::Ready(Ok(()))
138 }
139
140 fn call(&mut self, req: http::Request<Body>) -> Self::Future {
141 let router = self.router.clone();
142 let query_params = req
143 .uri()
144 .query()
145 .map(|q| {
146 url::form_urlencoded::parse(q.as_bytes())
147 .into_owned()
148 .collect()
149 })
150 .unwrap_or_default();
151 let path = req.uri().path().to_owned();
152 let method = req.method().clone();
153 Box::pin(async move { router.call(&path, method, query_params).await })
154 }
155}
156
157#[derive(Default)]
158pub struct Router {
159 handlers: HashMap<String, Arc<dyn HttpHandler>>,
160}
161
162impl Router {
163 pub fn new() -> Self {
164 Self {
165 handlers: HashMap::default(),
166 }
167 }
168
169 pub fn nest(path: &str, router: Router) -> Self {
170 check_path(path);
171
172 let handlers = router
173 .handlers
174 .into_iter()
175 .map(|(url, handler)| (format!("{path}{url}"), handler))
176 .collect();
177
178 Self { handlers }
179 }
180
181 pub fn route(mut self, path: &str, handler: impl HttpHandler + 'static) -> Self {
182 check_path(path);
183
184 let _ = self.handlers.insert(path.to_owned(), Arc::new(handler));
185
186 self
187 }
188
189 pub fn routes(mut self, paths: &[&str], handler: impl HttpHandler + 'static) -> Self {
190 let handler = Arc::new(handler);
191 for path in paths {
192 check_path(path);
193 let _ = self.handlers.insert(path.to_string(), handler.clone());
194 }
195
196 self
197 }
198
199 pub async fn call(
200 &self,
201 path: &str,
202 method: http::Method,
203 params: HashMap<String, String>,
204 ) -> Result<http::Response<Body>, Infallible> {
205 let handler = match self.handlers.get(path) {
206 Some(handler) => handler,
207 None => {
208 return Ok(http::Response::builder()
209 .status(http::StatusCode::NOT_FOUND)
210 .body(Body::empty())
211 .unwrap());
212 }
213 };
214
215 let res = match handler.handle(path, method, ¶ms).await {
216 Ok(res) => res.map(Body::new),
217 Err(e) => http::Response::builder()
218 .status(http::StatusCode::INTERNAL_SERVER_ERROR)
219 .body(Body::new(e.to_string()))
220 .unwrap(),
221 };
222
223 Ok(res)
224 }
225}
226
227fn check_path(path: &str) {
228 if path.is_empty() || !path.starts_with('/') {
229 panic!("paths must start with a `/`")
230 }
231}
232
233pub fn admin_axum_router(metasrv: Arc<Metasrv>) -> AxumRouter {
235 let node_lease_handler = NodeLeaseHandler {
236 meta_peer_client: metasrv.meta_peer_client().clone(),
237 };
238 let heartbeat_handler = HeartBeatHandler {
239 meta_peer_client: metasrv.meta_peer_client().clone(),
240 };
241 let leader_handler = LeaderHandler {
242 election: metasrv.election().cloned(),
243 };
244 let maintenance_handler = MaintenanceHandler {
245 manager: metasrv.runtime_switch_manager().clone(),
246 };
247 let procedure_handler = ProcedureManagerHandler {
248 manager: metasrv.runtime_switch_manager().clone(),
249 };
250 let recovery_handler = RecoveryHandler {
251 manager: metasrv.runtime_switch_manager().clone(),
252 };
253 let table_id_allocator_handler = TableIdAllocatorHandler {
254 table_id_allocator: metasrv.table_id_allocator().clone(),
255 runtime_switch_manager: metasrv.runtime_switch_manager().clone(),
256 };
257
258 let admin_router = AxumRouter::new()
259 .nest(
260 "/datanode",
261 AxumRouter::new()
262 .route("/drain", routing::post(datanode::drain))
263 .route(
264 "/mark-schedulable",
265 routing::post(datanode::mark_schedulable),
266 )
267 .route(
268 "/scheduling/status",
269 routing::get(datanode::scheduling_status),
270 )
271 .with_state(metasrv.clone()),
272 )
273 .route(
274 "/procedure/status",
275 routing::get(procedure::procedure_status).with_state(metasrv),
276 )
277 .route("/health", routing::get(health::health))
278 .route(
279 "/node-lease",
280 routing::get(node_lease::get).with_state(node_lease_handler),
281 )
282 .route(
283 "/leader",
284 routing::get(leader::get).with_state(leader_handler),
285 )
286 .nest(
287 "/heartbeat",
288 AxumRouter::new()
289 .route("/", routing::get(heartbeat::get))
290 .route("/help", routing::get(heartbeat::help))
291 .with_state(heartbeat_handler),
292 )
293 .nest(
294 "/maintenance",
295 AxumRouter::new()
296 .route("/", routing::get(maintenance::status))
297 .route("/status", routing::get(maintenance::status))
298 .route("/enable", routing::post(maintenance::set))
299 .route("/disable", routing::post(maintenance::unset))
300 .with_state(maintenance_handler),
301 )
302 .nest(
303 "/procedure-manager",
304 AxumRouter::new()
305 .route("/status", routing::get(procedure::status))
306 .route("/pause", routing::post(procedure::pause))
307 .route("/resume", routing::post(procedure::resume))
308 .with_state(procedure_handler),
309 )
310 .nest(
311 "/recovery",
312 AxumRouter::new()
313 .route("/status", routing::get(recovery::status))
314 .route("/enable", routing::post(recovery::set))
315 .route("/disable", routing::post(recovery::unset))
316 .with_state(recovery_handler),
317 )
318 .nest(
319 "/sequence",
320 AxumRouter::new().nest(
321 "/table",
322 AxumRouter::new()
323 .route("/next-id", routing::get(sequencer::get_next_table_id))
324 .route("/set-next-id", routing::post(sequencer::set_next_table_id))
325 .with_state(table_id_allocator_handler.clone()),
326 ),
327 );
328
329 AxumRouter::new().nest("/admin", admin_router)
330}
331
332#[cfg(test)]
333mod tests {
334 use common_meta::kv_backend::KvBackendRef;
335 use common_meta::kv_backend::memory::MemoryKvBackend;
336 use tokio::io::{AsyncReadExt, AsyncWriteExt, DuplexStream};
337
338 use super::*;
339 use crate::metasrv::MetasrvOptions;
340 use crate::metasrv::builder::MetasrvBuilder;
341 use crate::{bootstrap, error};
342
343 struct MockOkHandler;
344
345 #[async_trait::async_trait]
346 impl HttpHandler for MockOkHandler {
347 async fn handle(
348 &self,
349 _: &str,
350 _: http::Method,
351 _: &HashMap<String, String>,
352 ) -> crate::Result<http::Response<String>> {
353 Ok(http::Response::builder()
354 .status(http::StatusCode::OK)
355 .body("Ok".to_string())
356 .unwrap())
357 }
358 }
359 struct MockEmptyKeyErrorHandler;
360
361 #[async_trait::async_trait]
362 impl HttpHandler for MockEmptyKeyErrorHandler {
363 async fn handle(
364 &self,
365 _: &str,
366 _: http::Method,
367 _: &HashMap<String, String>,
368 ) -> crate::Result<http::Response<String>> {
369 error::EmptyKeySnafu {}.fail()
370 }
371 }
372
373 #[test]
374 fn test_route_nest() {
375 let mock_handler = MockOkHandler {};
376 let router = Router::new().route("/test_node", mock_handler);
377 let router = Router::nest("/test_root", router);
378
379 assert_eq!(1, router.handlers.len());
380 assert!(router.handlers.contains_key("/test_root/test_node"));
381 }
382
383 #[should_panic]
384 #[test]
385 fn test_invalid_path() {
386 check_path("test_node")
387 }
388
389 #[should_panic]
390 #[test]
391 fn test_empty_path() {
392 check_path("")
393 }
394
395 #[tokio::test]
396 async fn test_route_call_ok() {
397 let mock_handler = MockOkHandler {};
398 let router = Router::new().route("/test_node", mock_handler);
399 let router = Router::nest("/test_root", router);
400
401 let res = router
402 .call(
403 "/test_root/test_node",
404 http::Method::GET,
405 HashMap::default(),
406 )
407 .await
408 .unwrap();
409
410 assert!(res.status().is_success());
411 }
412
413 #[tokio::test]
414 async fn test_route_call_no_handler() {
415 let router = Router::new();
416
417 let res = router
418 .call(
419 "/test_root/test_node",
420 http::Method::GET,
421 HashMap::default(),
422 )
423 .await
424 .unwrap();
425
426 assert_eq!(http::StatusCode::NOT_FOUND, res.status());
427 }
428
429 #[tokio::test]
430 async fn test_route_call_err() {
431 let mock_handler = MockEmptyKeyErrorHandler {};
432 let router = Router::new().route("/test_node", mock_handler);
433 let router = Router::nest("/test_root", router);
434
435 let res = router
436 .call(
437 "/test_root/test_node",
438 http::Method::GET,
439 HashMap::default(),
440 )
441 .await
442 .unwrap();
443
444 assert_eq!(http::StatusCode::INTERNAL_SERVER_ERROR, res.status());
445 }
446
447 async fn test_metasrv(kv_backend: KvBackendRef) -> Metasrv {
448 let opts = MetasrvOptions::default();
449 let builder = MetasrvBuilder::new()
450 .options(opts)
451 .kv_backend(kv_backend.clone());
452
453 builder.build().await.unwrap()
454 }
455
456 async fn send_request(client: &mut DuplexStream, request: &[u8]) -> String {
457 client.write_all(request).await.unwrap();
458 let mut buf = vec![0; 1024];
459 let n = client.read(&mut buf).await.unwrap();
460 String::from_utf8_lossy(&buf[..n]).to_string()
461 }
462
463 #[tokio::test(flavor = "multi_thread")]
464 async fn test_metasrv_maintenance_mode() {
465 common_telemetry::init_default_ut_logging();
466 let kv_backend = Arc::new(MemoryKvBackend::new());
467 let metasrv = test_metasrv(kv_backend).await;
468 metasrv.try_start().await.unwrap();
469
470 let (mut client, server) = tokio::io::duplex(1024);
471 let metasrv = Arc::new(metasrv);
472 let service = metasrv.clone();
473 let _handle = tokio::spawn(async move {
474 let router = bootstrap::router(service);
475 router
476 .serve_with_incoming(futures::stream::iter(vec![Ok::<_, std::io::Error>(server)]))
477 .await
478 });
479
480 let response = send_request(
482 &mut client,
483 b"GET /admin/maintenance HTTP/1.1\r\nHost: localhost\r\n\r\n",
484 )
485 .await;
486 assert!(response.contains(r#"{"enabled":false}"#));
487 assert!(response.contains("200 OK"));
488
489 let response = send_request(
491 &mut client,
492 b"POST /admin/maintenance?enable=true HTTP/1.1\r\nHost: localhost\r\nContent-Length: 0\r\n\r\n",
493 )
494 .await;
495 assert!(response.contains(r#"{"enabled":true}"#));
496 assert!(response.contains("200 OK"));
497
498 let enabled = metasrv
499 .runtime_switch_manager()
500 .maintenance_mode()
501 .await
502 .unwrap();
503 assert!(enabled);
504
505 let response = send_request(
507 &mut client,
508 b"GET /admin/maintenance HTTP/1.1\r\nHost: localhost\r\n\r\n",
509 )
510 .await;
511 assert!(response.contains(r#"{"enabled":true}"#));
512 assert!(response.contains("200 OK"));
513
514 let response = send_request(
516 &mut client,
517 b"POST /admin/maintenance?enable=false HTTP/1.1\r\nHost: localhost\r\nContent-Length: 0\r\n\r\n",
518 )
519 .await;
520 assert!(response.contains(r#"{"enabled":false}"#));
521 assert!(response.contains("200 OK"));
522
523 let enabled = metasrv
524 .runtime_switch_manager()
525 .maintenance_mode()
526 .await
527 .unwrap();
528 assert!(!enabled);
529
530 let response = send_request(
532 &mut client,
533 b"GET /admin/maintenance?enable=true HTTP/1.1\r\nHost: localhost\r\n\r\n",
534 )
535 .await;
536 assert!(response.contains(r#"{"enabled":true}"#));
537 assert!(response.contains("200 OK"));
538
539 let response = send_request(
541 &mut client,
542 b"PUT /admin/maintenance?enable=false HTTP/1.1\r\nHost: localhost\r\n\r\n",
543 )
544 .await;
545 assert!(response.contains(r#"{"enabled":false}"#));
546 assert!(response.contains("200 OK"));
547
548 let response = send_request(
550 &mut client,
551 b"GET /admin/maintenance/status HTTP/1.1\r\nHost: localhost\r\n\r\n",
552 )
553 .await;
554 assert!(response.contains(r#"{"enabled":false}"#));
555
556 let response = send_request(
558 &mut client,
559 b"POST /admin/maintenance/enable HTTP/1.1\r\nHost: localhost\r\nContent-Length: 0\r\n\r\n",
560 )
561 .await;
562 assert!(response.contains(r#"{"enabled":true}"#));
563
564 let response = send_request(
566 &mut client,
567 b"POST /admin/maintenance/disable HTTP/1.1\r\nHost: localhost\r\nContent-Length: 0\r\n\r\n",
568 )
569 .await;
570 assert!(response.contains(r#"{"enabled":false}"#));
571
572 let response = send_request(
574 &mut client,
575 b"POST /admin/maintenance/status HTTP/1.1\r\nHost: localhost\r\nContent-Length: 0\r\n\r\n",
576 )
577 .await;
578 assert!(response.contains("404 Not Found"));
579
580 let response = send_request(
582 &mut client,
583 b"GET /admin/maintenance/enable HTTP/1.1\r\nHost: localhost\r\nContent-Length: 0\r\n\r\n",
584 )
585 .await;
586 assert!(response.contains("404 Not Found"));
587
588 let response = send_request(
590 &mut client,
591 b"GET /admin/maintenance/disable HTTP/1.1\r\nHost: localhost\r\nContent-Length: 0\r\n\r\n",
592 )
593 .await;
594 assert!(response.contains("404 Not Found"));
595 }
596
597 #[tokio::test(flavor = "multi_thread")]
598 async fn test_metasrv_procedure_manager_handler() {
599 common_telemetry::init_default_ut_logging();
600 let kv_backend = Arc::new(MemoryKvBackend::new());
601 let metasrv = test_metasrv(kv_backend).await;
602 metasrv.try_start().await.unwrap();
603
604 let (mut client, server) = tokio::io::duplex(1024);
605 let metasrv = Arc::new(metasrv);
606 let service = metasrv.clone();
607 let _handle = tokio::spawn(async move {
608 let router = bootstrap::router(service);
609 router
610 .serve_with_incoming(futures::stream::iter(vec![Ok::<_, std::io::Error>(server)]))
611 .await
612 });
613
614 let response = send_request(
616 &mut client,
617 b"GET /admin/procedure-manager/status HTTP/1.1\r\nHost: localhost\r\n\r\n",
618 )
619 .await;
620 assert!(response.contains("200 OK"));
621 assert!(
622 response.contains(r#"{"status":"running"}"#),
623 "response: {}",
624 response
625 );
626
627 let response = send_request(
629 &mut client,
630 b"POST /admin/procedure-manager/pause HTTP/1.1\r\nHost: localhost\r\n\r\n",
631 )
632 .await;
633 assert!(response.contains("200 OK"));
634 assert!(response.contains(r#"{"status":"paused"}"#));
635
636 let response = send_request(
638 &mut client,
639 b"POST /admin/procedure-manager/resume HTTP/1.1\r\nHost: localhost\r\n\r\n",
640 )
641 .await;
642 assert!(response.contains("200 OK"));
643 assert!(
644 response.contains(r#"{"status":"running"}"#),
645 "response: {}",
646 response
647 );
648
649 let response = send_request(
651 &mut client,
652 b"GET /admin/procedure-manager/resume HTTP/1.1\r\nHost: localhost\r\n\r\n",
653 )
654 .await;
655 assert!(response.contains("404 Not Found"));
656
657 let response = send_request(
659 &mut client,
660 b"GET /admin/procedure-manager/pause HTTP/1.1\r\nHost: localhost\r\n\r\n",
661 )
662 .await;
663 assert!(response.contains("404 Not Found"));
664 }
665}
666
667#[cfg(test)]
668mod axum_admin_tests {
669 use std::sync::Arc;
670
671 use axum::body::{Body, to_bytes};
672 use axum::http::{Method, Request, StatusCode};
673 use common_meta::kv_backend::memory::MemoryKvBackend;
674 use tower::ServiceExt; use super::*;
677 use crate::metasrv::MetasrvOptions;
678 use crate::metasrv::builder::MetasrvBuilder;
679 use crate::service::admin::sequencer::NextTableIdResponse;
680
681 async fn setup_axum_app() -> AxumRouter {
682 let kv_backend = Arc::new(MemoryKvBackend::new());
683 let metasrv = MetasrvBuilder::new()
684 .options(MetasrvOptions::default())
685 .kv_backend(kv_backend)
686 .build()
687 .await
688 .unwrap();
689 let metasrv = Arc::new(metasrv);
690 admin_axum_router(metasrv)
691 }
692
693 async fn get_body_string(resp: axum::response::Response) -> String {
694 let body_bytes = to_bytes(resp.into_body(), usize::MAX).await.unwrap();
695 String::from_utf8_lossy(&body_bytes).to_string()
696 }
697
698 async fn into_bytes(resp: axum::response::Response) -> Vec<u8> {
699 let body_bytes = to_bytes(resp.into_body(), usize::MAX).await.unwrap();
700 body_bytes.to_vec()
701 }
702
703 #[tokio::test]
704 async fn test_admin_health() {
705 let app = setup_axum_app().await;
706 let response = app
707 .oneshot(
708 Request::builder()
709 .uri("/admin/health")
710 .body(Body::empty())
711 .unwrap(),
712 )
713 .await
714 .unwrap();
715 assert_eq!(response.status(), StatusCode::OK);
716 let body = get_body_string(response).await;
717 assert!(body.to_lowercase().contains("ok"));
718 }
719
720 #[tokio::test]
721 async fn test_admin_node_lease() {
722 let app = setup_axum_app().await;
723 let response = app
724 .oneshot(
725 Request::builder()
726 .uri("/admin/node-lease")
727 .body(Body::empty())
728 .unwrap(),
729 )
730 .await
731 .unwrap();
732 assert_eq!(response.status(), StatusCode::OK);
733 }
734
735 #[tokio::test]
736 async fn test_admin_heartbeat() {
737 let app = setup_axum_app().await;
738 let response = app
739 .oneshot(
740 Request::builder()
741 .uri("/admin/heartbeat")
742 .body(Body::empty())
743 .unwrap(),
744 )
745 .await
746 .unwrap();
747 assert_eq!(response.status(), StatusCode::OK);
748 }
749
750 #[tokio::test]
751 async fn test_admin_heartbeat_help() {
752 let app = setup_axum_app().await;
753 let response = app
754 .oneshot(
755 Request::builder()
756 .uri("/admin/heartbeat/help")
757 .body(Body::empty())
758 .unwrap(),
759 )
760 .await
761 .unwrap();
762 assert_eq!(response.status(), StatusCode::OK);
763 }
764
765 #[tokio::test]
766 async fn test_admin_leader() {
767 let app = setup_axum_app().await;
768 let response = app
769 .oneshot(
770 Request::builder()
771 .uri("/admin/leader")
772 .body(Body::empty())
773 .unwrap(),
774 )
775 .await
776 .unwrap();
777 assert_eq!(response.status(), StatusCode::OK);
778 }
779
780 #[tokio::test]
781 async fn test_admin_maintenance() {
782 let app = setup_axum_app().await;
783 let response = app
784 .oneshot(
785 Request::builder()
786 .uri("/admin/maintenance")
787 .body(Body::empty())
788 .unwrap(),
789 )
790 .await
791 .unwrap();
792 assert_eq!(response.status(), StatusCode::OK);
793 let body = get_body_string(response).await;
794 assert!(body.contains("enabled"));
795 }
796
797 #[tokio::test]
798 async fn test_admin_maintenance_status() {
799 let app = setup_axum_app().await;
800 let response = app
801 .oneshot(
802 Request::builder()
803 .uri("/admin/maintenance/status")
804 .body(Body::empty())
805 .unwrap(),
806 )
807 .await
808 .unwrap();
809 assert_eq!(response.status(), StatusCode::OK);
810 let body = get_body_string(response).await;
811 assert!(body.contains("enabled"));
812 }
813
814 #[tokio::test]
815 async fn test_admin_maintenance_enable_disable() {
816 let response = setup_axum_app()
818 .await
819 .oneshot(
820 Request::builder()
821 .method(Method::POST)
822 .uri("/admin/maintenance/enable")
823 .body(Body::empty())
824 .unwrap(),
825 )
826 .await
827 .unwrap();
828 assert_eq!(response.status(), StatusCode::OK);
829 let body = get_body_string(response).await;
830 assert!(body.contains("enabled"));
831 let response = setup_axum_app()
833 .await
834 .oneshot(
835 Request::builder()
836 .method(Method::POST)
837 .uri("/admin/maintenance/disable")
838 .body(Body::empty())
839 .unwrap(),
840 )
841 .await
842 .unwrap();
843 assert_eq!(response.status(), StatusCode::OK);
844 let body = get_body_string(response).await;
845 assert!(body.contains("enabled"));
846 }
847
848 #[tokio::test]
849 async fn test_admin_procedure_manager_status() {
850 let app = setup_axum_app().await;
851 let response = app
852 .oneshot(
853 Request::builder()
854 .uri("/admin/procedure-manager/status")
855 .body(Body::empty())
856 .unwrap(),
857 )
858 .await
859 .unwrap();
860 assert_eq!(response.status(), StatusCode::OK);
861 let body = get_body_string(response).await;
862 assert!(body.contains("status"));
863 }
864
865 #[tokio::test]
866 async fn test_admin_procedure_manager_pause_resume() {
867 let response = setup_axum_app()
869 .await
870 .oneshot(
871 Request::builder()
872 .method(Method::POST)
873 .uri("/admin/procedure-manager/pause")
874 .body(Body::empty())
875 .unwrap(),
876 )
877 .await
878 .unwrap();
879 assert_eq!(response.status(), StatusCode::OK);
880 let body = get_body_string(response).await;
881 assert!(body.contains("paused"));
882 let response = setup_axum_app()
884 .await
885 .oneshot(
886 Request::builder()
887 .method(Method::POST)
888 .uri("/admin/procedure-manager/resume")
889 .body(Body::empty())
890 .unwrap(),
891 )
892 .await
893 .unwrap();
894 assert_eq!(response.status(), StatusCode::OK);
895 let body = get_body_string(response).await;
896 assert!(body.contains("running"));
897 }
898
899 #[tokio::test]
900 async fn test_admin_recovery() {
901 let app = setup_axum_app().await;
902 let response = app
903 .clone()
904 .oneshot(
905 Request::builder()
906 .uri("/admin/recovery/status")
907 .method(Method::GET)
908 .body(Body::empty())
909 .unwrap(),
910 )
911 .await
912 .unwrap();
913 assert_eq!(response.status(), StatusCode::OK);
914 let body = get_body_string(response).await;
915 assert!(body.contains("false"));
916
917 let response = app
919 .clone()
920 .oneshot(
921 Request::builder()
922 .uri("/admin/recovery/enable")
923 .method(Method::POST)
924 .body(Body::empty())
925 .unwrap(),
926 )
927 .await
928 .unwrap();
929 assert_eq!(response.status(), StatusCode::OK);
930 let body = get_body_string(response).await;
931 assert!(body.contains("true"));
932
933 let response = app
934 .clone()
935 .oneshot(
936 Request::builder()
937 .uri("/admin/recovery/status")
938 .method(Method::GET)
939 .body(Body::empty())
940 .unwrap(),
941 )
942 .await
943 .unwrap();
944 assert_eq!(response.status(), StatusCode::OK);
945 let body = get_body_string(response).await;
946 assert!(body.contains("true"));
947
948 let response = app
950 .clone()
951 .oneshot(
952 Request::builder()
953 .uri("/admin/recovery/disable")
954 .method(Method::POST)
955 .body(Body::empty())
956 .unwrap(),
957 )
958 .await
959 .unwrap();
960 assert_eq!(response.status(), StatusCode::OK);
961 let body = get_body_string(response).await;
962 assert!(body.contains("false"));
963
964 let response = app
965 .clone()
966 .oneshot(
967 Request::builder()
968 .uri("/admin/recovery/status")
969 .method(Method::GET)
970 .body(Body::empty())
971 .unwrap(),
972 )
973 .await
974 .unwrap();
975 assert_eq!(response.status(), StatusCode::OK);
976 let body = get_body_string(response).await;
977 assert!(body.contains("false"));
978 }
979
980 #[tokio::test]
981 async fn test_admin_sequence_table_id() {
982 common_telemetry::init_default_ut_logging();
983 let kv_backend = Arc::new(MemoryKvBackend::new());
984 let metasrv = MetasrvBuilder::new()
985 .options(MetasrvOptions::default())
986 .kv_backend(kv_backend)
987 .build()
988 .await
989 .unwrap();
990 let metasrv = Arc::new(metasrv);
991 let runtime_switch_manager = metasrv.runtime_switch_manager().clone();
992 let app = admin_axum_router(metasrv);
993 runtime_switch_manager.set_recovery_mode().await.unwrap();
995 let response = app
996 .clone()
997 .oneshot(
998 Request::builder()
999 .method(Method::GET)
1000 .uri("/admin/sequence/table/next-id")
1001 .body(Body::empty())
1002 .unwrap(),
1003 )
1004 .await
1005 .unwrap();
1006 assert_eq!(response.status(), StatusCode::OK);
1007 let body = into_bytes(response).await;
1008 let resp: NextTableIdResponse = serde_json::from_slice(&body).unwrap();
1009 assert_eq!(resp.next_table_id, 1024);
1010
1011 let response = app
1013 .clone()
1014 .oneshot(
1015 Request::builder()
1016 .method(Method::POST)
1017 .header(http::header::CONTENT_TYPE, "application/json")
1018 .uri("/admin/sequence/table/set-next-id")
1019 .body(Body::empty())
1020 .unwrap(),
1021 )
1022 .await
1023 .unwrap();
1024 assert_eq!(response.status(), StatusCode::BAD_REQUEST);
1025
1026 let response = app
1028 .clone()
1029 .oneshot(
1030 Request::builder()
1031 .method(Method::POST)
1032 .header(http::header::CONTENT_TYPE, "application/json")
1033 .uri("/admin/sequence/table/set-next-id")
1034 .body(Body::from(r#"{"next_table_id": 0}"#))
1035 .unwrap(),
1036 )
1037 .await
1038 .unwrap();
1039 assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
1040 let body = get_body_string(response).await;
1041 assert!(body.contains("is not greater than the current next value"));
1042
1043 let response = app
1045 .clone()
1046 .oneshot(
1047 Request::builder()
1048 .method(Method::POST)
1049 .header(http::header::CONTENT_TYPE, "application/json")
1050 .uri("/admin/sequence/table/set-next-id")
1051 .body(Body::from(r#"{"next_table_id": 2048}"#))
1052 .unwrap(),
1053 )
1054 .await
1055 .unwrap();
1056 assert_eq!(response.status(), StatusCode::OK);
1057
1058 let response = app
1060 .clone()
1061 .oneshot(
1062 Request::builder()
1063 .method(Method::GET)
1064 .uri("/admin/sequence/table/next-id")
1065 .body(Body::empty())
1066 .unwrap(),
1067 )
1068 .await
1069 .unwrap();
1070 assert_eq!(response.status(), StatusCode::OK);
1071 let body = into_bytes(response).await;
1072 let resp: NextTableIdResponse = serde_json::from_slice(&body).unwrap();
1073 assert_eq!(resp.next_table_id, 2048);
1074
1075 runtime_switch_manager.unset_recovery_mode().await.unwrap();
1077 let response = app
1079 .clone()
1080 .oneshot(
1081 Request::builder()
1082 .method(Method::POST)
1083 .header(http::header::CONTENT_TYPE, "application/json")
1084 .uri("/admin/sequence/table/set-next-id")
1085 .body(Body::from(r#"{"next_table_id": 2049}"#))
1086 .unwrap(),
1087 )
1088 .await
1089 .unwrap();
1090 assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
1091 let body = get_body_string(response).await;
1092 assert!(body.contains("Setting next table id is only allowed in recovery mode"));
1093 }
1094}