Skip to main content

meta_srv/service/admin/
datanode.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use std::sync::Arc;
16use std::time::Duration;
17
18use axum::Json;
19use axum::extract::{Query, State};
20use axum::response::{IntoResponse, Response};
21use common_procedure::ProcedureContext;
22use serde::{Deserialize, Serialize};
23use servers::http::result::error_result::ErrorResponse;
24
25use crate::key::NodeSchedulingRole;
26use crate::metasrv::Metasrv;
27use crate::procedure::drain_datanode::{
28    DEFAULT_DRAIN_TIMEOUT, DEFAULT_PARALLELISM, DrainDatanodeProcedureTask, DrainDatanodeStrategy,
29};
30use crate::service::admin::util;
31
32#[derive(Debug, Deserialize)]
33pub(crate) struct DrainDatanodeRequest {
34    node_id: u64,
35    target_node_id: Option<u64>,
36    /// Concurrent region budget; defaults to [`DEFAULT_PARALLELISM`].
37    #[serde(default = "default_parallelism")]
38    parallelism: usize,
39    /// Failure handling; defaults to [`DrainDatanodeStrategy::TryAll`].
40    #[serde(default)]
41    strategy: DrainStrategy,
42    /// Whole-drain budget; defaults to [`DEFAULT_DRAIN_TIMEOUT`].
43    #[serde(default = "default_timeout", with = "humantime_serde")]
44    timeout: Duration,
45}
46
47/// HTTP strategy names follow the admin API's snake_case convention.
48#[derive(Debug, Default, Deserialize)]
49#[serde(rename_all = "snake_case")]
50enum DrainStrategy {
51    FailFast,
52    #[default]
53    TryAll,
54}
55
56impl From<DrainStrategy> for DrainDatanodeStrategy {
57    fn from(strategy: DrainStrategy) -> Self {
58        match strategy {
59            DrainStrategy::FailFast => Self::FailFast,
60            DrainStrategy::TryAll => Self::TryAll,
61        }
62    }
63}
64
65impl From<DrainDatanodeRequest> for DrainDatanodeProcedureTask {
66    fn from(request: DrainDatanodeRequest) -> Self {
67        Self {
68            node_id: request.node_id,
69            target_node_id: request.target_node_id,
70            parallelism: request.parallelism,
71            strategy: request.strategy.into(),
72            timeout: Some(request.timeout),
73        }
74    }
75}
76
77#[derive(Debug, Deserialize)]
78pub(crate) struct DatanodeRequest {
79    node_id: u64,
80}
81
82#[derive(Debug, Serialize)]
83pub(crate) struct DrainDatanodeResponse {
84    procedure_id: String,
85}
86
87#[derive(Debug, Serialize)]
88pub(crate) struct DatanodeSchedulingResponse {
89    node_id: u64,
90    schedulable: bool,
91}
92
93fn default_parallelism() -> usize {
94    DEFAULT_PARALLELISM
95}
96
97fn default_timeout() -> Duration {
98    DEFAULT_DRAIN_TIMEOUT
99}
100
101/// Submits an asynchronous leader-region drain.
102/// Rejects requests for a datanode already being drained.
103#[axum_macros::debug_handler]
104pub(crate) async fn drain(
105    State(metasrv): State<Arc<Metasrv>>,
106    Json(request): Json<DrainDatanodeRequest>,
107) -> Response {
108    async {
109        util::ensure_leader(&metasrv)?;
110        let procedure_id = metasrv
111            .drain_datanode_manager()
112            .submit_procedure(ProcedureContext::default(), request.into())
113            .await?;
114        Ok::<_, crate::error::Error>(DrainDatanodeResponse {
115            procedure_id: procedure_id.to_string(),
116        })
117    }
118    .await
119    .map(Json)
120    .map_err(ErrorResponse::from_error)
121    .into_response()
122}
123
124/// Restores scheduling; an active drain must finish before the node can receive regions.
125#[axum_macros::debug_handler]
126pub(crate) async fn mark_schedulable(
127    State(metasrv): State<Arc<Metasrv>>,
128    Json(request): Json<DatanodeRequest>,
129) -> Response {
130    async {
131        util::ensure_leader(&metasrv)?;
132        metasrv
133            .drain_datanode_manager()
134            .mark_schedulable(request.node_id)
135            .await?;
136        Ok::<_, crate::error::Error>(DatanodeSchedulingResponse {
137            node_id: request.node_id,
138            schedulable: true,
139        })
140    }
141    .await
142    .map(Json)
143    .map_err(ErrorResponse::from_error)
144    .into_response()
145}
146
147/// Reads the node's scheduling state from the leader.
148#[axum_macros::debug_handler]
149pub(crate) async fn scheduling_status(
150    State(metasrv): State<Arc<Metasrv>>,
151    Query(request): Query<DatanodeRequest>,
152) -> Response {
153    async {
154        util::ensure_leader(&metasrv)?;
155        let schedulable = metasrv
156            .node_scheduling_manager()
157            .is_schedulable(NodeSchedulingRole::Datanode, request.node_id)
158            .await?;
159        Ok::<_, crate::error::Error>(DatanodeSchedulingResponse {
160            node_id: request.node_id,
161            schedulable,
162        })
163    }
164    .await
165    .map(Json)
166    .map_err(ErrorResponse::from_error)
167    .into_response()
168}
169
170#[cfg(test)]
171mod tests {
172    use std::sync::atomic::{AtomicBool, Ordering};
173
174    use axum::Router;
175    use axum::body::{Body, to_bytes};
176    use axum::http::{Method, Request, StatusCode};
177    use common_meta::election::{Election, LeaderChangeMessage, LeaderValue, MetasrvNodeInfo};
178    use serde_json::{Value, json};
179    use tower::ServiceExt;
180
181    use super::*;
182    use crate::metasrv::builder::MetasrvBuilder;
183    use crate::service::admin::admin_axum_router;
184
185    struct TestElection(AtomicBool);
186
187    #[async_trait::async_trait]
188    impl Election for TestElection {
189        type Leader = LeaderValue;
190        fn is_leader(&self) -> bool {
191            self.0.load(Ordering::Relaxed)
192        }
193        fn in_leader_infancy(&self) -> bool {
194            false
195        }
196        async fn register_candidate(&self, _: &MetasrvNodeInfo) -> common_meta::error::Result<()> {
197            Ok(())
198        }
199        async fn all_candidates(&self) -> common_meta::error::Result<Vec<MetasrvNodeInfo>> {
200            Ok(vec![])
201        }
202        async fn campaign(&self) -> common_meta::error::Result<()> {
203            Ok(())
204        }
205        async fn leader(&self) -> common_meta::error::Result<LeaderValue> {
206            Ok(LeaderValue("leader:3002".into()))
207        }
208        async fn resign(&self) -> common_meta::error::Result<()> {
209            Ok(())
210        }
211        fn subscribe_leader_change(&self) -> tokio::sync::broadcast::Receiver<LeaderChangeMessage> {
212            tokio::sync::broadcast::channel(1).1
213        }
214    }
215
216    async fn request(app: Router, method: Method, uri: &str, body: Value) -> (StatusCode, Value) {
217        let response = app
218            .oneshot(
219                Request::builder()
220                    .method(method)
221                    .uri(uri)
222                    .header("content-type", "application/json")
223                    .body(Body::from(body.to_string()))
224                    .unwrap(),
225            )
226            .await
227            .unwrap();
228        let status = response.status();
229        let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap();
230        (status, serde_json::from_slice(&bytes).unwrap())
231    }
232
233    #[test]
234    fn test_drain_request_options() {
235        for (body, parallelism, strategy) in [
236            (json!({"node_id": 1}), 4, DrainDatanodeStrategy::TryAll),
237            (
238                json!({"node_id": 1, "parallelism": 8}),
239                8,
240                DrainDatanodeStrategy::TryAll,
241            ),
242            (
243                json!({"node_id": 1, "strategy": "fail_fast"}),
244                4,
245                DrainDatanodeStrategy::FailFast,
246            ),
247            (
248                json!({"node_id": 1, "parallelism": 1, "strategy": "fail_fast"}),
249                1,
250                DrainDatanodeStrategy::FailFast,
251            ),
252            (
253                json!({"node_id": 1, "parallelism": 32, "strategy": "try_all"}),
254                32,
255                DrainDatanodeStrategy::TryAll,
256            ),
257        ] {
258            let request: DrainDatanodeRequest = serde_json::from_value(body).unwrap();
259            let task: DrainDatanodeProcedureTask = request.into();
260            assert_eq!(parallelism, task.parallelism);
261            assert_eq!(strategy, task.strategy);
262            assert_eq!(1, task.node_id);
263            assert_eq!(None, task.target_node_id);
264            assert_eq!(Some(Duration::from_secs(20 * 60)), task.timeout);
265        }
266    }
267
268    #[test]
269    fn test_drain_request_timeout() {
270        for (value, expected) in [
271            (json!("30m"), Some(Duration::from_secs(1800))),
272            (json!("1h 30m"), Some(Duration::from_secs(5400))),
273            (json!("500ms"), Some(Duration::from_millis(500))),
274        ] {
275            let request: DrainDatanodeRequest =
276                serde_json::from_value(json!({"node_id": 1, "timeout": value})).unwrap();
277            let task: DrainDatanodeProcedureTask = request.into();
278            assert_eq!(expected, task.timeout);
279        }
280    }
281
282    #[tokio::test]
283    async fn test_http_drain_rejects_invalid_options() {
284        let metasrv = Arc::new(MetasrvBuilder::new().build().await.unwrap());
285        metasrv.procedure_manager().start().await.unwrap();
286        let app = admin_axum_router(metasrv.clone());
287        for (options, expected) in [
288            (json!({"timeout": null}), StatusCode::UNPROCESSABLE_ENTITY),
289            (
290                json!({"parallelism": null}),
291                StatusCode::UNPROCESSABLE_ENTITY,
292            ),
293            (json!({"strategy": null}), StatusCode::BAD_REQUEST),
294            (json!({"timeout": "0s"}), StatusCode::BAD_REQUEST),
295            (
296                json!({"timeout": "9223372036854775s"}),
297                StatusCode::BAD_REQUEST,
298            ),
299            (
300                json!({"timeout": "9223372036854776s"}),
301                StatusCode::BAD_REQUEST,
302            ),
303            (
304                json!({"timeout": "invalid"}),
305                StatusCode::UNPROCESSABLE_ENTITY,
306            ),
307            (json!({"timeout": "-1s"}), StatusCode::UNPROCESSABLE_ENTITY),
308            (json!({"parallelism": 0}), StatusCode::BAD_REQUEST),
309            (json!({"parallelism": 33}), StatusCode::BAD_REQUEST),
310            (json!({"parallelism": -1}), StatusCode::UNPROCESSABLE_ENTITY),
311            (
312                json!({"parallelism": 1.5}),
313                StatusCode::UNPROCESSABLE_ENTITY,
314            ),
315            (
316                json!({"strategy": "unknown"}),
317                StatusCode::UNPROCESSABLE_ENTITY,
318            ),
319            (
320                json!({"strategy": "FailFast"}),
321                StatusCode::UNPROCESSABLE_ENTITY,
322            ),
323        ] {
324            let mut body = options;
325            body["node_id"] = json!(1);
326            let response = app
327                .clone()
328                .oneshot(
329                    Request::builder()
330                        .method(Method::POST)
331                        .uri("/admin/datanode/drain")
332                        .header("content-type", "application/json")
333                        .body(Body::from(body.to_string()))
334                        .unwrap(),
335                )
336                .await
337                .unwrap();
338            assert_eq!(expected, response.status(), "{body}");
339        }
340        assert!(
341            metasrv
342                .procedure_manager()
343                .list_procedures()
344                .await
345                .unwrap()
346                .is_empty()
347        );
348        assert!(
349            metasrv
350                .node_scheduling_manager()
351                .is_schedulable(NodeSchedulingRole::Datanode, 1)
352                .await
353                .unwrap()
354        );
355        metasrv.procedure_manager().stop().await.unwrap();
356    }
357
358    #[tokio::test]
359    async fn test_http_scheduling_is_idempotent() {
360        let metasrv = Arc::new(MetasrvBuilder::new().build().await.unwrap());
361        let app = admin_axum_router(metasrv.clone());
362        metasrv.procedure_manager().start().await.unwrap();
363        metasrv
364            .node_scheduling_manager()
365            .mark_unschedulable(NodeSchedulingRole::Datanode, 1)
366            .await
367            .unwrap();
368        for _ in 0..2 {
369            assert_eq!(
370                (StatusCode::OK, json!({"node_id": 1, "schedulable": true})),
371                request(
372                    app.clone(),
373                    Method::POST,
374                    "/admin/datanode/mark-schedulable",
375                    json!({"node_id": 1})
376                )
377                .await
378            );
379        }
380        assert_eq!(
381            (StatusCode::OK, json!({"node_id": 1, "schedulable": true})),
382            request(
383                app.clone(),
384                Method::GET,
385                "/admin/datanode/scheduling/status?node_id=1",
386                Value::Null
387            )
388            .await
389        );
390        let (status, _) = request(
391            app,
392            Method::POST,
393            "/admin/datanode/drain",
394            json!({"node_id": 1, "target_node_id": 1}),
395        )
396        .await;
397        assert_eq!(StatusCode::BAD_REQUEST, status);
398        metasrv.procedure_manager().stop().await.unwrap();
399    }
400
401    #[tokio::test]
402    async fn test_http_follower_rejects_admin_operations_without_side_effects() {
403        let election = Arc::new(TestElection(AtomicBool::new(false)));
404        let metasrv = Arc::new(
405            MetasrvBuilder::new()
406                .election(Some(election.clone()))
407                .build()
408                .await
409                .unwrap(),
410        );
411        let app = admin_axum_router(metasrv.clone());
412        metasrv.procedure_manager().start().await.unwrap();
413        metasrv
414            .node_scheduling_manager()
415            .mark_unschedulable(NodeSchedulingRole::Datanode, 1)
416            .await
417            .unwrap();
418        for (method, uri, body) in [
419            (
420                Method::POST,
421                "/admin/datanode/drain",
422                json!({"node_id": 1, "target_node_id": 2}),
423            ),
424            (
425                Method::POST,
426                "/admin/datanode/mark-schedulable",
427                json!({"node_id": 1}),
428            ),
429            (
430                Method::GET,
431                "/admin/datanode/scheduling/status?node_id=1",
432                Value::Null,
433            ),
434            (
435                Method::GET,
436                "/admin/procedure/status?procedure_id=00000000-0000-0000-0000-000000000001",
437                Value::Null,
438            ),
439        ] {
440            let (status, body) = request(app.clone(), method, uri, body).await;
441            assert_eq!(StatusCode::INTERNAL_SERVER_ERROR, status, "{body}");
442            assert!(body["error"].as_str().unwrap().contains("not leader"));
443        }
444        assert!(
445            metasrv
446                .procedure_manager()
447                .list_procedures()
448                .await
449                .unwrap()
450                .is_empty()
451        );
452        assert!(
453            !metasrv
454                .node_scheduling_manager()
455                .is_schedulable(NodeSchedulingRole::Datanode, 1)
456                .await
457                .unwrap()
458        );
459        election.0.store(true, Ordering::Relaxed);
460        assert_eq!(
461            (StatusCode::OK, json!({"node_id": 1, "schedulable": true})),
462            request(
463                app,
464                Method::POST,
465                "/admin/datanode/mark-schedulable",
466                json!({"node_id": 1})
467            )
468            .await
469        );
470        metasrv.procedure_manager().stop().await.unwrap();
471    }
472
473    #[tokio::test]
474    async fn test_http_procedure_status_and_request_validation() {
475        let metasrv = Arc::new(MetasrvBuilder::new().build().await.unwrap());
476        metasrv.procedure_manager().start().await.unwrap();
477        let app = admin_axum_router(metasrv.clone());
478        let (status, body) = request(
479            app.clone(),
480            Method::POST,
481            "/admin/datanode/drain",
482            json!({"node_id": 1}),
483        )
484        .await;
485        assert_eq!(StatusCode::OK, status, "{body}");
486        let pid = body["procedure_id"].as_str().unwrap();
487        let uri = format!("/admin/procedure/status?procedure_id={pid}");
488        tokio::time::timeout(std::time::Duration::from_secs(5), async {
489            loop {
490                let (status, body) = request(app.clone(), Method::GET, &uri, Value::Null).await;
491                assert_eq!(StatusCode::OK, status);
492                assert_eq!(pid, body["procedure_id"]);
493                if body["status"] == "Done" {
494                    assert_eq!(
495                        json!({"procedure_id": pid, "status": "Done", "error": null}),
496                        body
497                    );
498                    break;
499                }
500                tokio::task::yield_now().await;
501            }
502        })
503        .await
504        .unwrap();
505        for (uri, expected) in [
506            (
507                "/admin/procedure/status?procedure_id=bad",
508                StatusCode::BAD_REQUEST,
509            ),
510            (
511                "/admin/procedure/status?procedure_id=00000000-0000-0000-0000-000000000001",
512                StatusCode::BAD_REQUEST,
513            ),
514        ] {
515            let (status, body) = request(app.clone(), Method::GET, uri, Value::Null).await;
516            assert_eq!(expected, status);
517            if uri.ends_with("=bad") {
518                assert!(
519                    body["error"]
520                        .as_str()
521                        .unwrap()
522                        .contains("invalid procedure id: bad")
523                );
524            }
525        }
526        for (method, uri, body, expected) in [
527            (
528                Method::GET,
529                "/admin/datanode/drain",
530                json!({"node_id": 1}),
531                StatusCode::METHOD_NOT_ALLOWED,
532            ),
533            (
534                Method::POST,
535                "/admin/datanode/drain",
536                json!({"node_id": -1}),
537                StatusCode::UNPROCESSABLE_ENTITY,
538            ),
539            (
540                Method::POST,
541                "/admin/datanode/drain",
542                json!({}),
543                StatusCode::UNPROCESSABLE_ENTITY,
544            ),
545            (
546                Method::GET,
547                "/admin/datanode/scheduling/status",
548                Value::Null,
549                StatusCode::BAD_REQUEST,
550            ),
551        ] {
552            let response = app
553                .clone()
554                .oneshot(
555                    Request::builder()
556                        .method(method)
557                        .uri(uri)
558                        .header("content-type", "application/json")
559                        .body(Body::from(body.to_string()))
560                        .unwrap(),
561                )
562                .await
563                .unwrap();
564            assert_eq!(expected, response.status());
565        }
566        metasrv.procedure_manager().stop().await.unwrap();
567    }
568}