1use std::sync::Arc;
16use std::time::Duration;
17
18use axum::Json;
19use axum::extract::{Query, State};
20use axum::response::{IntoResponse, Response};
21use common_procedure::ProcedureContext;
22use serde::{Deserialize, Serialize};
23use servers::http::result::error_result::ErrorResponse;
24
25use crate::key::NodeSchedulingRole;
26use crate::metasrv::Metasrv;
27use crate::procedure::drain_datanode::{
28 DEFAULT_DRAIN_TIMEOUT, DEFAULT_PARALLELISM, DrainDatanodeProcedureTask, DrainDatanodeStrategy,
29};
30use crate::service::admin::util;
31
32#[derive(Debug, Deserialize)]
33pub(crate) struct DrainDatanodeRequest {
34 node_id: u64,
35 target_node_id: Option<u64>,
36 #[serde(default = "default_parallelism")]
38 parallelism: usize,
39 #[serde(default)]
41 strategy: DrainStrategy,
42 #[serde(default = "default_timeout", with = "humantime_serde")]
44 timeout: Duration,
45}
46
47#[derive(Debug, Default, Deserialize)]
49#[serde(rename_all = "snake_case")]
50enum DrainStrategy {
51 FailFast,
52 #[default]
53 TryAll,
54}
55
56impl From<DrainStrategy> for DrainDatanodeStrategy {
57 fn from(strategy: DrainStrategy) -> Self {
58 match strategy {
59 DrainStrategy::FailFast => Self::FailFast,
60 DrainStrategy::TryAll => Self::TryAll,
61 }
62 }
63}
64
65impl From<DrainDatanodeRequest> for DrainDatanodeProcedureTask {
66 fn from(request: DrainDatanodeRequest) -> Self {
67 Self {
68 node_id: request.node_id,
69 target_node_id: request.target_node_id,
70 parallelism: request.parallelism,
71 strategy: request.strategy.into(),
72 timeout: Some(request.timeout),
73 }
74 }
75}
76
77#[derive(Debug, Deserialize)]
78pub(crate) struct DatanodeRequest {
79 node_id: u64,
80}
81
82#[derive(Debug, Serialize)]
83pub(crate) struct DrainDatanodeResponse {
84 procedure_id: String,
85}
86
87#[derive(Debug, Serialize)]
88pub(crate) struct DatanodeSchedulingResponse {
89 node_id: u64,
90 schedulable: bool,
91}
92
93fn default_parallelism() -> usize {
94 DEFAULT_PARALLELISM
95}
96
97fn default_timeout() -> Duration {
98 DEFAULT_DRAIN_TIMEOUT
99}
100
101#[axum_macros::debug_handler]
104pub(crate) async fn drain(
105 State(metasrv): State<Arc<Metasrv>>,
106 Json(request): Json<DrainDatanodeRequest>,
107) -> Response {
108 async {
109 util::ensure_leader(&metasrv)?;
110 let procedure_id = metasrv
111 .drain_datanode_manager()
112 .submit_procedure(ProcedureContext::default(), request.into())
113 .await?;
114 Ok::<_, crate::error::Error>(DrainDatanodeResponse {
115 procedure_id: procedure_id.to_string(),
116 })
117 }
118 .await
119 .map(Json)
120 .map_err(ErrorResponse::from_error)
121 .into_response()
122}
123
124#[axum_macros::debug_handler]
126pub(crate) async fn mark_schedulable(
127 State(metasrv): State<Arc<Metasrv>>,
128 Json(request): Json<DatanodeRequest>,
129) -> Response {
130 async {
131 util::ensure_leader(&metasrv)?;
132 metasrv
133 .drain_datanode_manager()
134 .mark_schedulable(request.node_id)
135 .await?;
136 Ok::<_, crate::error::Error>(DatanodeSchedulingResponse {
137 node_id: request.node_id,
138 schedulable: true,
139 })
140 }
141 .await
142 .map(Json)
143 .map_err(ErrorResponse::from_error)
144 .into_response()
145}
146
147#[axum_macros::debug_handler]
149pub(crate) async fn scheduling_status(
150 State(metasrv): State<Arc<Metasrv>>,
151 Query(request): Query<DatanodeRequest>,
152) -> Response {
153 async {
154 util::ensure_leader(&metasrv)?;
155 let schedulable = metasrv
156 .node_scheduling_manager()
157 .is_schedulable(NodeSchedulingRole::Datanode, request.node_id)
158 .await?;
159 Ok::<_, crate::error::Error>(DatanodeSchedulingResponse {
160 node_id: request.node_id,
161 schedulable,
162 })
163 }
164 .await
165 .map(Json)
166 .map_err(ErrorResponse::from_error)
167 .into_response()
168}
169
170#[cfg(test)]
171mod tests {
172 use std::sync::atomic::{AtomicBool, Ordering};
173
174 use axum::Router;
175 use axum::body::{Body, to_bytes};
176 use axum::http::{Method, Request, StatusCode};
177 use common_meta::election::{Election, LeaderChangeMessage, LeaderValue, MetasrvNodeInfo};
178 use serde_json::{Value, json};
179 use tower::ServiceExt;
180
181 use super::*;
182 use crate::metasrv::builder::MetasrvBuilder;
183 use crate::service::admin::admin_axum_router;
184
185 struct TestElection(AtomicBool);
186
187 #[async_trait::async_trait]
188 impl Election for TestElection {
189 type Leader = LeaderValue;
190 fn is_leader(&self) -> bool {
191 self.0.load(Ordering::Relaxed)
192 }
193 fn in_leader_infancy(&self) -> bool {
194 false
195 }
196 async fn register_candidate(&self, _: &MetasrvNodeInfo) -> common_meta::error::Result<()> {
197 Ok(())
198 }
199 async fn all_candidates(&self) -> common_meta::error::Result<Vec<MetasrvNodeInfo>> {
200 Ok(vec![])
201 }
202 async fn campaign(&self) -> common_meta::error::Result<()> {
203 Ok(())
204 }
205 async fn leader(&self) -> common_meta::error::Result<LeaderValue> {
206 Ok(LeaderValue("leader:3002".into()))
207 }
208 async fn resign(&self) -> common_meta::error::Result<()> {
209 Ok(())
210 }
211 fn subscribe_leader_change(&self) -> tokio::sync::broadcast::Receiver<LeaderChangeMessage> {
212 tokio::sync::broadcast::channel(1).1
213 }
214 }
215
216 async fn request(app: Router, method: Method, uri: &str, body: Value) -> (StatusCode, Value) {
217 let response = app
218 .oneshot(
219 Request::builder()
220 .method(method)
221 .uri(uri)
222 .header("content-type", "application/json")
223 .body(Body::from(body.to_string()))
224 .unwrap(),
225 )
226 .await
227 .unwrap();
228 let status = response.status();
229 let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap();
230 (status, serde_json::from_slice(&bytes).unwrap())
231 }
232
233 #[test]
234 fn test_drain_request_options() {
235 for (body, parallelism, strategy) in [
236 (json!({"node_id": 1}), 4, DrainDatanodeStrategy::TryAll),
237 (
238 json!({"node_id": 1, "parallelism": 8}),
239 8,
240 DrainDatanodeStrategy::TryAll,
241 ),
242 (
243 json!({"node_id": 1, "strategy": "fail_fast"}),
244 4,
245 DrainDatanodeStrategy::FailFast,
246 ),
247 (
248 json!({"node_id": 1, "parallelism": 1, "strategy": "fail_fast"}),
249 1,
250 DrainDatanodeStrategy::FailFast,
251 ),
252 (
253 json!({"node_id": 1, "parallelism": 32, "strategy": "try_all"}),
254 32,
255 DrainDatanodeStrategy::TryAll,
256 ),
257 ] {
258 let request: DrainDatanodeRequest = serde_json::from_value(body).unwrap();
259 let task: DrainDatanodeProcedureTask = request.into();
260 assert_eq!(parallelism, task.parallelism);
261 assert_eq!(strategy, task.strategy);
262 assert_eq!(1, task.node_id);
263 assert_eq!(None, task.target_node_id);
264 assert_eq!(Some(Duration::from_secs(20 * 60)), task.timeout);
265 }
266 }
267
268 #[test]
269 fn test_drain_request_timeout() {
270 for (value, expected) in [
271 (json!("30m"), Some(Duration::from_secs(1800))),
272 (json!("1h 30m"), Some(Duration::from_secs(5400))),
273 (json!("500ms"), Some(Duration::from_millis(500))),
274 ] {
275 let request: DrainDatanodeRequest =
276 serde_json::from_value(json!({"node_id": 1, "timeout": value})).unwrap();
277 let task: DrainDatanodeProcedureTask = request.into();
278 assert_eq!(expected, task.timeout);
279 }
280 }
281
282 #[tokio::test]
283 async fn test_http_drain_rejects_invalid_options() {
284 let metasrv = Arc::new(MetasrvBuilder::new().build().await.unwrap());
285 metasrv.procedure_manager().start().await.unwrap();
286 let app = admin_axum_router(metasrv.clone());
287 for (options, expected) in [
288 (json!({"timeout": null}), StatusCode::UNPROCESSABLE_ENTITY),
289 (
290 json!({"parallelism": null}),
291 StatusCode::UNPROCESSABLE_ENTITY,
292 ),
293 (json!({"strategy": null}), StatusCode::BAD_REQUEST),
294 (json!({"timeout": "0s"}), StatusCode::BAD_REQUEST),
295 (
296 json!({"timeout": "9223372036854775s"}),
297 StatusCode::BAD_REQUEST,
298 ),
299 (
300 json!({"timeout": "9223372036854776s"}),
301 StatusCode::BAD_REQUEST,
302 ),
303 (
304 json!({"timeout": "invalid"}),
305 StatusCode::UNPROCESSABLE_ENTITY,
306 ),
307 (json!({"timeout": "-1s"}), StatusCode::UNPROCESSABLE_ENTITY),
308 (json!({"parallelism": 0}), StatusCode::BAD_REQUEST),
309 (json!({"parallelism": 33}), StatusCode::BAD_REQUEST),
310 (json!({"parallelism": -1}), StatusCode::UNPROCESSABLE_ENTITY),
311 (
312 json!({"parallelism": 1.5}),
313 StatusCode::UNPROCESSABLE_ENTITY,
314 ),
315 (
316 json!({"strategy": "unknown"}),
317 StatusCode::UNPROCESSABLE_ENTITY,
318 ),
319 (
320 json!({"strategy": "FailFast"}),
321 StatusCode::UNPROCESSABLE_ENTITY,
322 ),
323 ] {
324 let mut body = options;
325 body["node_id"] = json!(1);
326 let response = app
327 .clone()
328 .oneshot(
329 Request::builder()
330 .method(Method::POST)
331 .uri("/admin/datanode/drain")
332 .header("content-type", "application/json")
333 .body(Body::from(body.to_string()))
334 .unwrap(),
335 )
336 .await
337 .unwrap();
338 assert_eq!(expected, response.status(), "{body}");
339 }
340 assert!(
341 metasrv
342 .procedure_manager()
343 .list_procedures()
344 .await
345 .unwrap()
346 .is_empty()
347 );
348 assert!(
349 metasrv
350 .node_scheduling_manager()
351 .is_schedulable(NodeSchedulingRole::Datanode, 1)
352 .await
353 .unwrap()
354 );
355 metasrv.procedure_manager().stop().await.unwrap();
356 }
357
358 #[tokio::test]
359 async fn test_http_scheduling_is_idempotent() {
360 let metasrv = Arc::new(MetasrvBuilder::new().build().await.unwrap());
361 let app = admin_axum_router(metasrv.clone());
362 metasrv.procedure_manager().start().await.unwrap();
363 metasrv
364 .node_scheduling_manager()
365 .mark_unschedulable(NodeSchedulingRole::Datanode, 1)
366 .await
367 .unwrap();
368 for _ in 0..2 {
369 assert_eq!(
370 (StatusCode::OK, json!({"node_id": 1, "schedulable": true})),
371 request(
372 app.clone(),
373 Method::POST,
374 "/admin/datanode/mark-schedulable",
375 json!({"node_id": 1})
376 )
377 .await
378 );
379 }
380 assert_eq!(
381 (StatusCode::OK, json!({"node_id": 1, "schedulable": true})),
382 request(
383 app.clone(),
384 Method::GET,
385 "/admin/datanode/scheduling/status?node_id=1",
386 Value::Null
387 )
388 .await
389 );
390 let (status, _) = request(
391 app,
392 Method::POST,
393 "/admin/datanode/drain",
394 json!({"node_id": 1, "target_node_id": 1}),
395 )
396 .await;
397 assert_eq!(StatusCode::BAD_REQUEST, status);
398 metasrv.procedure_manager().stop().await.unwrap();
399 }
400
401 #[tokio::test]
402 async fn test_http_follower_rejects_admin_operations_without_side_effects() {
403 let election = Arc::new(TestElection(AtomicBool::new(false)));
404 let metasrv = Arc::new(
405 MetasrvBuilder::new()
406 .election(Some(election.clone()))
407 .build()
408 .await
409 .unwrap(),
410 );
411 let app = admin_axum_router(metasrv.clone());
412 metasrv.procedure_manager().start().await.unwrap();
413 metasrv
414 .node_scheduling_manager()
415 .mark_unschedulable(NodeSchedulingRole::Datanode, 1)
416 .await
417 .unwrap();
418 for (method, uri, body) in [
419 (
420 Method::POST,
421 "/admin/datanode/drain",
422 json!({"node_id": 1, "target_node_id": 2}),
423 ),
424 (
425 Method::POST,
426 "/admin/datanode/mark-schedulable",
427 json!({"node_id": 1}),
428 ),
429 (
430 Method::GET,
431 "/admin/datanode/scheduling/status?node_id=1",
432 Value::Null,
433 ),
434 (
435 Method::GET,
436 "/admin/procedure/status?procedure_id=00000000-0000-0000-0000-000000000001",
437 Value::Null,
438 ),
439 ] {
440 let (status, body) = request(app.clone(), method, uri, body).await;
441 assert_eq!(StatusCode::INTERNAL_SERVER_ERROR, status, "{body}");
442 assert!(body["error"].as_str().unwrap().contains("not leader"));
443 }
444 assert!(
445 metasrv
446 .procedure_manager()
447 .list_procedures()
448 .await
449 .unwrap()
450 .is_empty()
451 );
452 assert!(
453 !metasrv
454 .node_scheduling_manager()
455 .is_schedulable(NodeSchedulingRole::Datanode, 1)
456 .await
457 .unwrap()
458 );
459 election.0.store(true, Ordering::Relaxed);
460 assert_eq!(
461 (StatusCode::OK, json!({"node_id": 1, "schedulable": true})),
462 request(
463 app,
464 Method::POST,
465 "/admin/datanode/mark-schedulable",
466 json!({"node_id": 1})
467 )
468 .await
469 );
470 metasrv.procedure_manager().stop().await.unwrap();
471 }
472
473 #[tokio::test]
474 async fn test_http_procedure_status_and_request_validation() {
475 let metasrv = Arc::new(MetasrvBuilder::new().build().await.unwrap());
476 metasrv.procedure_manager().start().await.unwrap();
477 let app = admin_axum_router(metasrv.clone());
478 let (status, body) = request(
479 app.clone(),
480 Method::POST,
481 "/admin/datanode/drain",
482 json!({"node_id": 1}),
483 )
484 .await;
485 assert_eq!(StatusCode::OK, status, "{body}");
486 let pid = body["procedure_id"].as_str().unwrap();
487 let uri = format!("/admin/procedure/status?procedure_id={pid}");
488 tokio::time::timeout(std::time::Duration::from_secs(5), async {
489 loop {
490 let (status, body) = request(app.clone(), Method::GET, &uri, Value::Null).await;
491 assert_eq!(StatusCode::OK, status);
492 assert_eq!(pid, body["procedure_id"]);
493 if body["status"] == "Done" {
494 assert_eq!(
495 json!({"procedure_id": pid, "status": "Done", "error": null}),
496 body
497 );
498 break;
499 }
500 tokio::task::yield_now().await;
501 }
502 })
503 .await
504 .unwrap();
505 for (uri, expected) in [
506 (
507 "/admin/procedure/status?procedure_id=bad",
508 StatusCode::BAD_REQUEST,
509 ),
510 (
511 "/admin/procedure/status?procedure_id=00000000-0000-0000-0000-000000000001",
512 StatusCode::BAD_REQUEST,
513 ),
514 ] {
515 let (status, body) = request(app.clone(), Method::GET, uri, Value::Null).await;
516 assert_eq!(expected, status);
517 if uri.ends_with("=bad") {
518 assert!(
519 body["error"]
520 .as_str()
521 .unwrap()
522 .contains("invalid procedure id: bad")
523 );
524 }
525 }
526 for (method, uri, body, expected) in [
527 (
528 Method::GET,
529 "/admin/datanode/drain",
530 json!({"node_id": 1}),
531 StatusCode::METHOD_NOT_ALLOWED,
532 ),
533 (
534 Method::POST,
535 "/admin/datanode/drain",
536 json!({"node_id": -1}),
537 StatusCode::UNPROCESSABLE_ENTITY,
538 ),
539 (
540 Method::POST,
541 "/admin/datanode/drain",
542 json!({}),
543 StatusCode::UNPROCESSABLE_ENTITY,
544 ),
545 (
546 Method::GET,
547 "/admin/datanode/scheduling/status",
548 Value::Null,
549 StatusCode::BAD_REQUEST,
550 ),
551 ] {
552 let response = app
553 .clone()
554 .oneshot(
555 Request::builder()
556 .method(method)
557 .uri(uri)
558 .header("content-type", "application/json")
559 .body(Body::from(body.to_string()))
560 .unwrap(),
561 )
562 .await
563 .unwrap();
564 assert_eq!(expected, response.status());
565 }
566 metasrv.procedure_manager().stop().await.unwrap();
567 }
568}