Skip to main content

meta_srv/service/
admin.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15pub(crate) mod datanode;
16pub(crate) mod health;
17pub(crate) mod heartbeat;
18pub(crate) mod leader;
19pub(crate) mod maintenance;
20pub(crate) mod node_lease;
21pub(crate) mod procedure;
22pub(crate) mod recovery;
23pub(crate) mod sequencer;
24mod util;
25
26use std::collections::HashMap;
27use std::convert::Infallible;
28use std::sync::Arc;
29use std::task::{Context, Poll};
30
31use axum::{Router as AxumRouter, routing};
32use tonic::body::Body;
33use tonic::codegen::{BoxFuture, Service, http};
34use tonic::server::NamedService;
35
36use crate::metasrv::Metasrv;
37use crate::service::admin::heartbeat::HeartBeatHandler;
38use crate::service::admin::leader::LeaderHandler;
39use crate::service::admin::maintenance::MaintenanceHandler;
40use crate::service::admin::node_lease::NodeLeaseHandler;
41use crate::service::admin::procedure::ProcedureManagerHandler;
42use crate::service::admin::recovery::RecoveryHandler;
43use crate::service::admin::sequencer::TableIdAllocatorHandler;
44
45/// Expose admin http service on rpc port(3002).
46///
47/// # Deprecated
48///
49/// This function is deprecated and will be removed in the future. Please use
50/// [`admin_axum_router`] instead.
51pub fn make_admin_service(metasrv: Arc<Metasrv>) -> Admin {
52    let router = Router::new().route("/health", health::HealthHandler);
53
54    let router = router.route(
55        "/node-lease",
56        node_lease::NodeLeaseHandler {
57            meta_peer_client: metasrv.meta_peer_client().clone(),
58        },
59    );
60
61    let handler = heartbeat::HeartBeatHandler {
62        meta_peer_client: metasrv.meta_peer_client().clone(),
63    };
64    let router = router
65        .route("/heartbeat", handler.clone())
66        .route("/heartbeat/help", handler);
67
68    let router = router.route(
69        "/leader",
70        leader::LeaderHandler {
71            election: metasrv.election().cloned(),
72        },
73    );
74
75    let router = router.routes(
76        &[
77            "/maintenance",
78            "/maintenance/status",
79            "/maintenance/enable",
80            "/maintenance/disable",
81        ],
82        maintenance::MaintenanceHandler {
83            manager: metasrv.runtime_switch_manager().clone(),
84        },
85    );
86    let router = router.routes(
87        &[
88            "/procedure-manager/pause",
89            "/procedure-manager/resume",
90            "/procedure-manager/status",
91        ],
92        procedure::ProcedureManagerHandler {
93            manager: metasrv.runtime_switch_manager().clone(),
94        },
95    );
96    let router = Router::nest("/admin", router);
97
98    Admin::new(router)
99}
100
101#[async_trait::async_trait]
102pub trait HttpHandler: Send + Sync {
103    async fn handle(
104        &self,
105        path: &str,
106        method: http::Method,
107        params: &HashMap<String, String>,
108    ) -> crate::Result<http::Response<String>>;
109}
110
111#[derive(Clone)]
112pub struct Admin
113where
114    Self: Send,
115{
116    router: Arc<Router>,
117}
118
119impl Admin {
120    pub fn new(router: Router) -> Self {
121        Self {
122            router: Arc::new(router),
123        }
124    }
125}
126
127impl NamedService for Admin {
128    const NAME: &'static str = "admin";
129}
130
131impl Service<http::Request<Body>> for Admin {
132    type Response = http::Response<Body>;
133    type Error = Infallible;
134    type Future = BoxFuture<Self::Response, Self::Error>;
135
136    fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
137        Poll::Ready(Ok(()))
138    }
139
140    fn call(&mut self, req: http::Request<Body>) -> Self::Future {
141        let router = self.router.clone();
142        let query_params = req
143            .uri()
144            .query()
145            .map(|q| {
146                url::form_urlencoded::parse(q.as_bytes())
147                    .into_owned()
148                    .collect()
149            })
150            .unwrap_or_default();
151        let path = req.uri().path().to_owned();
152        let method = req.method().clone();
153        Box::pin(async move { router.call(&path, method, query_params).await })
154    }
155}
156
157#[derive(Default)]
158pub struct Router {
159    handlers: HashMap<String, Arc<dyn HttpHandler>>,
160}
161
162impl Router {
163    pub fn new() -> Self {
164        Self {
165            handlers: HashMap::default(),
166        }
167    }
168
169    pub fn nest(path: &str, router: Router) -> Self {
170        check_path(path);
171
172        let handlers = router
173            .handlers
174            .into_iter()
175            .map(|(url, handler)| (format!("{path}{url}"), handler))
176            .collect();
177
178        Self { handlers }
179    }
180
181    pub fn route(mut self, path: &str, handler: impl HttpHandler + 'static) -> Self {
182        check_path(path);
183
184        let _ = self.handlers.insert(path.to_owned(), Arc::new(handler));
185
186        self
187    }
188
189    pub fn routes(mut self, paths: &[&str], handler: impl HttpHandler + 'static) -> Self {
190        let handler = Arc::new(handler);
191        for path in paths {
192            check_path(path);
193            let _ = self.handlers.insert(path.to_string(), handler.clone());
194        }
195
196        self
197    }
198
199    pub async fn call(
200        &self,
201        path: &str,
202        method: http::Method,
203        params: HashMap<String, String>,
204    ) -> Result<http::Response<Body>, Infallible> {
205        let handler = match self.handlers.get(path) {
206            Some(handler) => handler,
207            None => {
208                return Ok(http::Response::builder()
209                    .status(http::StatusCode::NOT_FOUND)
210                    .body(Body::empty())
211                    .unwrap());
212            }
213        };
214
215        let res = match handler.handle(path, method, &params).await {
216            Ok(res) => res.map(Body::new),
217            Err(e) => http::Response::builder()
218                .status(http::StatusCode::INTERNAL_SERVER_ERROR)
219                .body(Body::new(e.to_string()))
220                .unwrap(),
221        };
222
223        Ok(res)
224    }
225}
226
227fn check_path(path: &str) {
228    if path.is_empty() || !path.starts_with('/') {
229        panic!("paths must start with a `/`")
230    }
231}
232
233/// Expose admin HTTP endpoints as an Axum router for the main HTTP server.
234pub fn admin_axum_router(metasrv: Arc<Metasrv>) -> AxumRouter {
235    let node_lease_handler = NodeLeaseHandler {
236        meta_peer_client: metasrv.meta_peer_client().clone(),
237    };
238    let heartbeat_handler = HeartBeatHandler {
239        meta_peer_client: metasrv.meta_peer_client().clone(),
240    };
241    let leader_handler = LeaderHandler {
242        election: metasrv.election().cloned(),
243    };
244    let maintenance_handler = MaintenanceHandler {
245        manager: metasrv.runtime_switch_manager().clone(),
246    };
247    let procedure_handler = ProcedureManagerHandler {
248        manager: metasrv.runtime_switch_manager().clone(),
249    };
250    let recovery_handler = RecoveryHandler {
251        manager: metasrv.runtime_switch_manager().clone(),
252    };
253    let table_id_allocator_handler = TableIdAllocatorHandler {
254        table_id_allocator: metasrv.table_id_allocator().clone(),
255        runtime_switch_manager: metasrv.runtime_switch_manager().clone(),
256    };
257
258    let admin_router = AxumRouter::new()
259        .nest(
260            "/datanode",
261            AxumRouter::new()
262                .route("/drain", routing::post(datanode::drain))
263                .route(
264                    "/mark-schedulable",
265                    routing::post(datanode::mark_schedulable),
266                )
267                .route(
268                    "/scheduling/status",
269                    routing::get(datanode::scheduling_status),
270                )
271                .with_state(metasrv.clone()),
272        )
273        .route(
274            "/procedure/status",
275            routing::get(procedure::procedure_status).with_state(metasrv),
276        )
277        .route("/health", routing::get(health::health))
278        .route(
279            "/node-lease",
280            routing::get(node_lease::get).with_state(node_lease_handler),
281        )
282        .route(
283            "/leader",
284            routing::get(leader::get).with_state(leader_handler),
285        )
286        .nest(
287            "/heartbeat",
288            AxumRouter::new()
289                .route("/", routing::get(heartbeat::get))
290                .route("/help", routing::get(heartbeat::help))
291                .with_state(heartbeat_handler),
292        )
293        .nest(
294            "/maintenance",
295            AxumRouter::new()
296                .route("/", routing::get(maintenance::status))
297                .route("/status", routing::get(maintenance::status))
298                .route("/enable", routing::post(maintenance::set))
299                .route("/disable", routing::post(maintenance::unset))
300                .with_state(maintenance_handler),
301        )
302        .nest(
303            "/procedure-manager",
304            AxumRouter::new()
305                .route("/status", routing::get(procedure::status))
306                .route("/pause", routing::post(procedure::pause))
307                .route("/resume", routing::post(procedure::resume))
308                .with_state(procedure_handler),
309        )
310        .nest(
311            "/recovery",
312            AxumRouter::new()
313                .route("/status", routing::get(recovery::status))
314                .route("/enable", routing::post(recovery::set))
315                .route("/disable", routing::post(recovery::unset))
316                .with_state(recovery_handler),
317        )
318        .nest(
319            "/sequence",
320            AxumRouter::new().nest(
321                "/table",
322                AxumRouter::new()
323                    .route("/next-id", routing::get(sequencer::get_next_table_id))
324                    .route("/set-next-id", routing::post(sequencer::set_next_table_id))
325                    .with_state(table_id_allocator_handler.clone()),
326            ),
327        );
328
329    AxumRouter::new().nest("/admin", admin_router)
330}
331
332#[cfg(test)]
333mod tests {
334    use common_meta::kv_backend::KvBackendRef;
335    use common_meta::kv_backend::memory::MemoryKvBackend;
336    use tokio::io::{AsyncReadExt, AsyncWriteExt, DuplexStream};
337
338    use super::*;
339    use crate::metasrv::MetasrvOptions;
340    use crate::metasrv::builder::MetasrvBuilder;
341    use crate::{bootstrap, error};
342
343    struct MockOkHandler;
344
345    #[async_trait::async_trait]
346    impl HttpHandler for MockOkHandler {
347        async fn handle(
348            &self,
349            _: &str,
350            _: http::Method,
351            _: &HashMap<String, String>,
352        ) -> crate::Result<http::Response<String>> {
353            Ok(http::Response::builder()
354                .status(http::StatusCode::OK)
355                .body("Ok".to_string())
356                .unwrap())
357        }
358    }
359    struct MockEmptyKeyErrorHandler;
360
361    #[async_trait::async_trait]
362    impl HttpHandler for MockEmptyKeyErrorHandler {
363        async fn handle(
364            &self,
365            _: &str,
366            _: http::Method,
367            _: &HashMap<String, String>,
368        ) -> crate::Result<http::Response<String>> {
369            error::EmptyKeySnafu {}.fail()
370        }
371    }
372
373    #[test]
374    fn test_route_nest() {
375        let mock_handler = MockOkHandler {};
376        let router = Router::new().route("/test_node", mock_handler);
377        let router = Router::nest("/test_root", router);
378
379        assert_eq!(1, router.handlers.len());
380        assert!(router.handlers.contains_key("/test_root/test_node"));
381    }
382
383    #[should_panic]
384    #[test]
385    fn test_invalid_path() {
386        check_path("test_node")
387    }
388
389    #[should_panic]
390    #[test]
391    fn test_empty_path() {
392        check_path("")
393    }
394
395    #[tokio::test]
396    async fn test_route_call_ok() {
397        let mock_handler = MockOkHandler {};
398        let router = Router::new().route("/test_node", mock_handler);
399        let router = Router::nest("/test_root", router);
400
401        let res = router
402            .call(
403                "/test_root/test_node",
404                http::Method::GET,
405                HashMap::default(),
406            )
407            .await
408            .unwrap();
409
410        assert!(res.status().is_success());
411    }
412
413    #[tokio::test]
414    async fn test_route_call_no_handler() {
415        let router = Router::new();
416
417        let res = router
418            .call(
419                "/test_root/test_node",
420                http::Method::GET,
421                HashMap::default(),
422            )
423            .await
424            .unwrap();
425
426        assert_eq!(http::StatusCode::NOT_FOUND, res.status());
427    }
428
429    #[tokio::test]
430    async fn test_route_call_err() {
431        let mock_handler = MockEmptyKeyErrorHandler {};
432        let router = Router::new().route("/test_node", mock_handler);
433        let router = Router::nest("/test_root", router);
434
435        let res = router
436            .call(
437                "/test_root/test_node",
438                http::Method::GET,
439                HashMap::default(),
440            )
441            .await
442            .unwrap();
443
444        assert_eq!(http::StatusCode::INTERNAL_SERVER_ERROR, res.status());
445    }
446
447    async fn test_metasrv(kv_backend: KvBackendRef) -> Metasrv {
448        let opts = MetasrvOptions::default();
449        let builder = MetasrvBuilder::new()
450            .options(opts)
451            .kv_backend(kv_backend.clone());
452
453        builder.build().await.unwrap()
454    }
455
456    async fn send_request(client: &mut DuplexStream, request: &[u8]) -> String {
457        client.write_all(request).await.unwrap();
458        let mut buf = vec![0; 1024];
459        let n = client.read(&mut buf).await.unwrap();
460        String::from_utf8_lossy(&buf[..n]).to_string()
461    }
462
463    #[tokio::test(flavor = "multi_thread")]
464    async fn test_metasrv_maintenance_mode() {
465        common_telemetry::init_default_ut_logging();
466        let kv_backend = Arc::new(MemoryKvBackend::new());
467        let metasrv = test_metasrv(kv_backend).await;
468        metasrv.try_start().await.unwrap();
469
470        let (mut client, server) = tokio::io::duplex(1024);
471        let metasrv = Arc::new(metasrv);
472        let service = metasrv.clone();
473        let _handle = tokio::spawn(async move {
474            let router = bootstrap::router(service);
475            router
476                .serve_with_incoming(futures::stream::iter(vec![Ok::<_, std::io::Error>(server)]))
477                .await
478        });
479
480        // Get maintenance mode
481        let response = send_request(
482            &mut client,
483            b"GET /admin/maintenance HTTP/1.1\r\nHost: localhost\r\n\r\n",
484        )
485        .await;
486        assert!(response.contains(r#"{"enabled":false}"#));
487        assert!(response.contains("200 OK"));
488
489        // Set maintenance mode to true
490        let response = send_request(
491            &mut client,
492            b"POST /admin/maintenance?enable=true HTTP/1.1\r\nHost: localhost\r\nContent-Length: 0\r\n\r\n",
493        )
494        .await;
495        assert!(response.contains(r#"{"enabled":true}"#));
496        assert!(response.contains("200 OK"));
497
498        let enabled = metasrv
499            .runtime_switch_manager()
500            .maintenance_mode()
501            .await
502            .unwrap();
503        assert!(enabled);
504
505        // Get maintenance mode again
506        let response = send_request(
507            &mut client,
508            b"GET /admin/maintenance HTTP/1.1\r\nHost: localhost\r\n\r\n",
509        )
510        .await;
511        assert!(response.contains(r#"{"enabled":true}"#));
512        assert!(response.contains("200 OK"));
513
514        // Set maintenance mode to false
515        let response = send_request(
516            &mut client,
517            b"POST /admin/maintenance?enable=false HTTP/1.1\r\nHost: localhost\r\nContent-Length: 0\r\n\r\n",
518        )
519        .await;
520        assert!(response.contains(r#"{"enabled":false}"#));
521        assert!(response.contains("200 OK"));
522
523        let enabled = metasrv
524            .runtime_switch_manager()
525            .maintenance_mode()
526            .await
527            .unwrap();
528        assert!(!enabled);
529
530        // Set maintenance mode to true via GET request
531        let response = send_request(
532            &mut client,
533            b"GET /admin/maintenance?enable=true HTTP/1.1\r\nHost: localhost\r\n\r\n",
534        )
535        .await;
536        assert!(response.contains(r#"{"enabled":true}"#));
537        assert!(response.contains("200 OK"));
538
539        // Set maintenance mode to false via GET request
540        let response = send_request(
541            &mut client,
542            b"PUT /admin/maintenance?enable=false HTTP/1.1\r\nHost: localhost\r\n\r\n",
543        )
544        .await;
545        assert!(response.contains(r#"{"enabled":false}"#));
546        assert!(response.contains("200 OK"));
547
548        // Get maintenance mode via status path
549        let response = send_request(
550            &mut client,
551            b"GET /admin/maintenance/status HTTP/1.1\r\nHost: localhost\r\n\r\n",
552        )
553        .await;
554        assert!(response.contains(r#"{"enabled":false}"#));
555
556        // Set maintenance mode via enable path
557        let response = send_request(
558            &mut client,
559            b"POST /admin/maintenance/enable HTTP/1.1\r\nHost: localhost\r\nContent-Length: 0\r\n\r\n",
560        )
561        .await;
562        assert!(response.contains(r#"{"enabled":true}"#));
563
564        // Unset maintenance mode via disable path
565        let response = send_request(
566            &mut client,
567            b"POST /admin/maintenance/disable HTTP/1.1\r\nHost: localhost\r\nContent-Length: 0\r\n\r\n",
568        )
569        .await;
570        assert!(response.contains(r#"{"enabled":false}"#));
571
572        // send POST request to status path
573        let response = send_request(
574            &mut client,
575            b"POST /admin/maintenance/status HTTP/1.1\r\nHost: localhost\r\nContent-Length: 0\r\n\r\n",
576        )
577        .await;
578        assert!(response.contains("404 Not Found"));
579
580        // send GET request to enable path
581        let response = send_request(
582            &mut client,
583            b"GET /admin/maintenance/enable HTTP/1.1\r\nHost: localhost\r\nContent-Length: 0\r\n\r\n",
584        )
585        .await;
586        assert!(response.contains("404 Not Found"));
587
588        // send GET request to disable path
589        let response = send_request(
590            &mut client,
591            b"GET /admin/maintenance/disable HTTP/1.1\r\nHost: localhost\r\nContent-Length: 0\r\n\r\n",
592        )
593        .await;
594        assert!(response.contains("404 Not Found"));
595    }
596
597    #[tokio::test(flavor = "multi_thread")]
598    async fn test_metasrv_procedure_manager_handler() {
599        common_telemetry::init_default_ut_logging();
600        let kv_backend = Arc::new(MemoryKvBackend::new());
601        let metasrv = test_metasrv(kv_backend).await;
602        metasrv.try_start().await.unwrap();
603
604        let (mut client, server) = tokio::io::duplex(1024);
605        let metasrv = Arc::new(metasrv);
606        let service = metasrv.clone();
607        let _handle = tokio::spawn(async move {
608            let router = bootstrap::router(service);
609            router
610                .serve_with_incoming(futures::stream::iter(vec![Ok::<_, std::io::Error>(server)]))
611                .await
612        });
613
614        // send GET request to procedure-manager/status path
615        let response = send_request(
616            &mut client,
617            b"GET /admin/procedure-manager/status HTTP/1.1\r\nHost: localhost\r\n\r\n",
618        )
619        .await;
620        assert!(response.contains("200 OK"));
621        assert!(
622            response.contains(r#"{"status":"running"}"#),
623            "response: {}",
624            response
625        );
626
627        // send POST request to procedure-manager/pause path
628        let response = send_request(
629            &mut client,
630            b"POST /admin/procedure-manager/pause HTTP/1.1\r\nHost: localhost\r\n\r\n",
631        )
632        .await;
633        assert!(response.contains("200 OK"));
634        assert!(response.contains(r#"{"status":"paused"}"#));
635
636        // send POST request to procedure-manager/resume path
637        let response = send_request(
638            &mut client,
639            b"POST /admin/procedure-manager/resume HTTP/1.1\r\nHost: localhost\r\n\r\n",
640        )
641        .await;
642        assert!(response.contains("200 OK"));
643        assert!(
644            response.contains(r#"{"status":"running"}"#),
645            "response: {}",
646            response
647        );
648
649        // send GET request to procedure-manager/resume path
650        let response = send_request(
651            &mut client,
652            b"GET /admin/procedure-manager/resume HTTP/1.1\r\nHost: localhost\r\n\r\n",
653        )
654        .await;
655        assert!(response.contains("404 Not Found"));
656
657        // send GET request to procedure-manager/pause path
658        let response = send_request(
659            &mut client,
660            b"GET /admin/procedure-manager/pause HTTP/1.1\r\nHost: localhost\r\n\r\n",
661        )
662        .await;
663        assert!(response.contains("404 Not Found"));
664    }
665}
666
667#[cfg(test)]
668mod axum_admin_tests {
669    use std::sync::Arc;
670
671    use axum::body::{Body, to_bytes};
672    use axum::http::{Method, Request, StatusCode};
673    use common_meta::kv_backend::memory::MemoryKvBackend;
674    use tower::ServiceExt; // for `oneshot`
675
676    use super::*;
677    use crate::metasrv::MetasrvOptions;
678    use crate::metasrv::builder::MetasrvBuilder;
679    use crate::service::admin::sequencer::NextTableIdResponse;
680
681    async fn setup_axum_app() -> AxumRouter {
682        let kv_backend = Arc::new(MemoryKvBackend::new());
683        let metasrv = MetasrvBuilder::new()
684            .options(MetasrvOptions::default())
685            .kv_backend(kv_backend)
686            .build()
687            .await
688            .unwrap();
689        let metasrv = Arc::new(metasrv);
690        admin_axum_router(metasrv)
691    }
692
693    async fn get_body_string(resp: axum::response::Response) -> String {
694        let body_bytes = to_bytes(resp.into_body(), usize::MAX).await.unwrap();
695        String::from_utf8_lossy(&body_bytes).to_string()
696    }
697
698    async fn into_bytes(resp: axum::response::Response) -> Vec<u8> {
699        let body_bytes = to_bytes(resp.into_body(), usize::MAX).await.unwrap();
700        body_bytes.to_vec()
701    }
702
703    #[tokio::test]
704    async fn test_admin_health() {
705        let app = setup_axum_app().await;
706        let response = app
707            .oneshot(
708                Request::builder()
709                    .uri("/admin/health")
710                    .body(Body::empty())
711                    .unwrap(),
712            )
713            .await
714            .unwrap();
715        assert_eq!(response.status(), StatusCode::OK);
716        let body = get_body_string(response).await;
717        assert!(body.to_lowercase().contains("ok"));
718    }
719
720    #[tokio::test]
721    async fn test_admin_node_lease() {
722        let app = setup_axum_app().await;
723        let response = app
724            .oneshot(
725                Request::builder()
726                    .uri("/admin/node-lease")
727                    .body(Body::empty())
728                    .unwrap(),
729            )
730            .await
731            .unwrap();
732        assert_eq!(response.status(), StatusCode::OK);
733    }
734
735    #[tokio::test]
736    async fn test_admin_heartbeat() {
737        let app = setup_axum_app().await;
738        let response = app
739            .oneshot(
740                Request::builder()
741                    .uri("/admin/heartbeat")
742                    .body(Body::empty())
743                    .unwrap(),
744            )
745            .await
746            .unwrap();
747        assert_eq!(response.status(), StatusCode::OK);
748    }
749
750    #[tokio::test]
751    async fn test_admin_heartbeat_help() {
752        let app = setup_axum_app().await;
753        let response = app
754            .oneshot(
755                Request::builder()
756                    .uri("/admin/heartbeat/help")
757                    .body(Body::empty())
758                    .unwrap(),
759            )
760            .await
761            .unwrap();
762        assert_eq!(response.status(), StatusCode::OK);
763    }
764
765    #[tokio::test]
766    async fn test_admin_leader() {
767        let app = setup_axum_app().await;
768        let response = app
769            .oneshot(
770                Request::builder()
771                    .uri("/admin/leader")
772                    .body(Body::empty())
773                    .unwrap(),
774            )
775            .await
776            .unwrap();
777        assert_eq!(response.status(), StatusCode::OK);
778    }
779
780    #[tokio::test]
781    async fn test_admin_maintenance() {
782        let app = setup_axum_app().await;
783        let response = app
784            .oneshot(
785                Request::builder()
786                    .uri("/admin/maintenance")
787                    .body(Body::empty())
788                    .unwrap(),
789            )
790            .await
791            .unwrap();
792        assert_eq!(response.status(), StatusCode::OK);
793        let body = get_body_string(response).await;
794        assert!(body.contains("enabled"));
795    }
796
797    #[tokio::test]
798    async fn test_admin_maintenance_status() {
799        let app = setup_axum_app().await;
800        let response = app
801            .oneshot(
802                Request::builder()
803                    .uri("/admin/maintenance/status")
804                    .body(Body::empty())
805                    .unwrap(),
806            )
807            .await
808            .unwrap();
809        assert_eq!(response.status(), StatusCode::OK);
810        let body = get_body_string(response).await;
811        assert!(body.contains("enabled"));
812    }
813
814    #[tokio::test]
815    async fn test_admin_maintenance_enable_disable() {
816        // Enable maintenance
817        let response = setup_axum_app()
818            .await
819            .oneshot(
820                Request::builder()
821                    .method(Method::POST)
822                    .uri("/admin/maintenance/enable")
823                    .body(Body::empty())
824                    .unwrap(),
825            )
826            .await
827            .unwrap();
828        assert_eq!(response.status(), StatusCode::OK);
829        let body = get_body_string(response).await;
830        assert!(body.contains("enabled"));
831        // Disable maintenance
832        let response = setup_axum_app()
833            .await
834            .oneshot(
835                Request::builder()
836                    .method(Method::POST)
837                    .uri("/admin/maintenance/disable")
838                    .body(Body::empty())
839                    .unwrap(),
840            )
841            .await
842            .unwrap();
843        assert_eq!(response.status(), StatusCode::OK);
844        let body = get_body_string(response).await;
845        assert!(body.contains("enabled"));
846    }
847
848    #[tokio::test]
849    async fn test_admin_procedure_manager_status() {
850        let app = setup_axum_app().await;
851        let response = app
852            .oneshot(
853                Request::builder()
854                    .uri("/admin/procedure-manager/status")
855                    .body(Body::empty())
856                    .unwrap(),
857            )
858            .await
859            .unwrap();
860        assert_eq!(response.status(), StatusCode::OK);
861        let body = get_body_string(response).await;
862        assert!(body.contains("status"));
863    }
864
865    #[tokio::test]
866    async fn test_admin_procedure_manager_pause_resume() {
867        // Pause
868        let response = setup_axum_app()
869            .await
870            .oneshot(
871                Request::builder()
872                    .method(Method::POST)
873                    .uri("/admin/procedure-manager/pause")
874                    .body(Body::empty())
875                    .unwrap(),
876            )
877            .await
878            .unwrap();
879        assert_eq!(response.status(), StatusCode::OK);
880        let body = get_body_string(response).await;
881        assert!(body.contains("paused"));
882        // Resume
883        let response = setup_axum_app()
884            .await
885            .oneshot(
886                Request::builder()
887                    .method(Method::POST)
888                    .uri("/admin/procedure-manager/resume")
889                    .body(Body::empty())
890                    .unwrap(),
891            )
892            .await
893            .unwrap();
894        assert_eq!(response.status(), StatusCode::OK);
895        let body = get_body_string(response).await;
896        assert!(body.contains("running"));
897    }
898
899    #[tokio::test]
900    async fn test_admin_recovery() {
901        let app = setup_axum_app().await;
902        let response = app
903            .clone()
904            .oneshot(
905                Request::builder()
906                    .uri("/admin/recovery/status")
907                    .method(Method::GET)
908                    .body(Body::empty())
909                    .unwrap(),
910            )
911            .await
912            .unwrap();
913        assert_eq!(response.status(), StatusCode::OK);
914        let body = get_body_string(response).await;
915        assert!(body.contains("false"));
916
917        // Enable recovery
918        let response = app
919            .clone()
920            .oneshot(
921                Request::builder()
922                    .uri("/admin/recovery/enable")
923                    .method(Method::POST)
924                    .body(Body::empty())
925                    .unwrap(),
926            )
927            .await
928            .unwrap();
929        assert_eq!(response.status(), StatusCode::OK);
930        let body = get_body_string(response).await;
931        assert!(body.contains("true"));
932
933        let response = app
934            .clone()
935            .oneshot(
936                Request::builder()
937                    .uri("/admin/recovery/status")
938                    .method(Method::GET)
939                    .body(Body::empty())
940                    .unwrap(),
941            )
942            .await
943            .unwrap();
944        assert_eq!(response.status(), StatusCode::OK);
945        let body = get_body_string(response).await;
946        assert!(body.contains("true"));
947
948        // Disable recovery
949        let response = app
950            .clone()
951            .oneshot(
952                Request::builder()
953                    .uri("/admin/recovery/disable")
954                    .method(Method::POST)
955                    .body(Body::empty())
956                    .unwrap(),
957            )
958            .await
959            .unwrap();
960        assert_eq!(response.status(), StatusCode::OK);
961        let body = get_body_string(response).await;
962        assert!(body.contains("false"));
963
964        let response = app
965            .clone()
966            .oneshot(
967                Request::builder()
968                    .uri("/admin/recovery/status")
969                    .method(Method::GET)
970                    .body(Body::empty())
971                    .unwrap(),
972            )
973            .await
974            .unwrap();
975        assert_eq!(response.status(), StatusCode::OK);
976        let body = get_body_string(response).await;
977        assert!(body.contains("false"));
978    }
979
980    #[tokio::test]
981    async fn test_admin_sequence_table_id() {
982        common_telemetry::init_default_ut_logging();
983        let kv_backend = Arc::new(MemoryKvBackend::new());
984        let metasrv = MetasrvBuilder::new()
985            .options(MetasrvOptions::default())
986            .kv_backend(kv_backend)
987            .build()
988            .await
989            .unwrap();
990        let metasrv = Arc::new(metasrv);
991        let runtime_switch_manager = metasrv.runtime_switch_manager().clone();
992        let app = admin_axum_router(metasrv);
993        // Set recovery mode to true
994        runtime_switch_manager.set_recovery_mode().await.unwrap();
995        let response = app
996            .clone()
997            .oneshot(
998                Request::builder()
999                    .method(Method::GET)
1000                    .uri("/admin/sequence/table/next-id")
1001                    .body(Body::empty())
1002                    .unwrap(),
1003            )
1004            .await
1005            .unwrap();
1006        assert_eq!(response.status(), StatusCode::OK);
1007        let body = into_bytes(response).await;
1008        let resp: NextTableIdResponse = serde_json::from_slice(&body).unwrap();
1009        assert_eq!(resp.next_table_id, 1024);
1010
1011        // Bad request
1012        let response = app
1013            .clone()
1014            .oneshot(
1015                Request::builder()
1016                    .method(Method::POST)
1017                    .header(http::header::CONTENT_TYPE, "application/json")
1018                    .uri("/admin/sequence/table/set-next-id")
1019                    .body(Body::empty())
1020                    .unwrap(),
1021            )
1022            .await
1023            .unwrap();
1024        assert_eq!(response.status(), StatusCode::BAD_REQUEST);
1025
1026        // Bad next id
1027        let response = app
1028            .clone()
1029            .oneshot(
1030                Request::builder()
1031                    .method(Method::POST)
1032                    .header(http::header::CONTENT_TYPE, "application/json")
1033                    .uri("/admin/sequence/table/set-next-id")
1034                    .body(Body::from(r#"{"next_table_id": 0}"#))
1035                    .unwrap(),
1036            )
1037            .await
1038            .unwrap();
1039        assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
1040        let body = get_body_string(response).await;
1041        assert!(body.contains("is not greater than the current next value"));
1042
1043        // Set next id
1044        let response = app
1045            .clone()
1046            .oneshot(
1047                Request::builder()
1048                    .method(Method::POST)
1049                    .header(http::header::CONTENT_TYPE, "application/json")
1050                    .uri("/admin/sequence/table/set-next-id")
1051                    .body(Body::from(r#"{"next_table_id": 2048}"#))
1052                    .unwrap(),
1053            )
1054            .await
1055            .unwrap();
1056        assert_eq!(response.status(), StatusCode::OK);
1057
1058        // Set next id
1059        let response = app
1060            .clone()
1061            .oneshot(
1062                Request::builder()
1063                    .method(Method::GET)
1064                    .uri("/admin/sequence/table/next-id")
1065                    .body(Body::empty())
1066                    .unwrap(),
1067            )
1068            .await
1069            .unwrap();
1070        assert_eq!(response.status(), StatusCode::OK);
1071        let body = into_bytes(response).await;
1072        let resp: NextTableIdResponse = serde_json::from_slice(&body).unwrap();
1073        assert_eq!(resp.next_table_id, 2048);
1074
1075        // Set recovery mode to false
1076        runtime_switch_manager.unset_recovery_mode().await.unwrap();
1077        // Set next id with recovery mode disabled
1078        let response = app
1079            .clone()
1080            .oneshot(
1081                Request::builder()
1082                    .method(Method::POST)
1083                    .header(http::header::CONTENT_TYPE, "application/json")
1084                    .uri("/admin/sequence/table/set-next-id")
1085                    .body(Body::from(r#"{"next_table_id": 2049}"#))
1086                    .unwrap(),
1087            )
1088            .await
1089            .unwrap();
1090        assert_eq!(response.status(), StatusCode::INTERNAL_SERVER_ERROR);
1091        let body = get_body_string(response).await;
1092        assert!(body.contains("Setting next table id is only allowed in recovery mode"));
1093    }
1094}