1mod trace_ingest;
16pub mod trace_semconv;
17pub mod trace_types;
18
19use std::collections::HashSet;
20use std::sync::Arc;
21
22use api::v1::RowInsertRequests;
23use api::v1::value::ValueData;
24use async_trait::async_trait;
25use auth::{
26 OTLP_WRITE, PermissionChecker, PermissionCheckerRef, PermissionReq, PermissionTableTarget,
27 PermissionTableTargets,
28};
29use client::Output;
30use common_catalog::consts::{
31 METRICS_EXEMPLARS_TABLE, trace_operations_table_name, trace_services_table_name,
32};
33use common_error::ext::BoxedError;
34use common_query::prelude::{EXEMPLAR_METRIC_COLUMN, GREPTIME_PHYSICAL_TABLE};
35use common_telemetry::{tracing, warn};
36use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest;
37use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
38use operator::insert::{Inserter, admit_row_insert_batches, admit_write};
39use otel_arrow_rust::proto::opentelemetry::collector::metrics::v1::ExportMetricsServiceRequest;
40use pipeline::{GreptimePipelineParams, PipelineWay};
41use servers::error::{self, AuthSnafu, Result as ServerResult};
42use servers::http::prom_store::PHYSICAL_TABLE_PARAM;
43use servers::interceptor::{OpenTelemetryProtocolInterceptor, OpenTelemetryProtocolInterceptorRef};
44use servers::otlp;
45use servers::otlp::trace::span::TraceSpanGroup;
46use servers::query_handler::{
47 MetricsIngestOutcome, OpenTelemetryProtocolHandler, PipelineHandlerRef, TraceIngestOutcome,
48};
49use servers::request_memory_limiter::ServerMemoryLimiter;
50use session::context::QueryContextRef;
51use session::protocol_ctx::OtlpMetricCtx;
52use session::write_progress::WriteProgress;
53use snafu::{IntoError, ResultExt};
54use table::requests::{
55 OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM, SEMANTIC_PER_TABLE_INDEX_KEY,
56 SEMANTIC_PIPELINE, SEMANTIC_SIGNAL_TYPE, SEMANTIC_SOURCE, SIGNAL_TYPE_LOG, SIGNAL_TYPE_METRIC,
57 SOURCE_OPENTELEMETRY, TABLE_DATA_MODEL, TABLE_DATA_MODEL_LOG_V1,
58};
59
60pub(crate) use self::trace_ingest::TraceAuxCache;
61use self::trace_ingest::trace_conventions;
62use crate::instance::Instance;
63use crate::metrics::{OTLP_LOGS_ROWS, OTLP_METRICS_ROWS, OTLP_RESOURCE_INFO_WRITE_ERRORS};
64
65fn trace_permission_targets(
66 table_name: &str,
67 groups: &[TraceSpanGroup],
68 ctx: &QueryContextRef,
69) -> PermissionTableTargets {
70 let catalog = ctx.current_catalog();
71 let schema = ctx.current_schema();
72 if catalog.is_empty() || schema.is_empty() || table_name.is_empty() {
73 return PermissionTableTargets::Unresolved;
74 }
75
76 let has_spans = groups.iter().any(|group| !group.spans.is_empty());
77 if !has_spans {
78 return PermissionTableTargets::resolved(Vec::new());
79 }
80
81 let mut targets = vec![PermissionTableTarget::new(catalog, &schema, table_name)];
82 if groups
83 .iter()
84 .flat_map(|group| &group.spans)
85 .any(|span| span.service_name.is_some())
86 {
87 targets.extend([
88 PermissionTableTarget::new(catalog, &schema, trace_services_table_name(table_name)),
89 PermissionTableTarget::new(catalog, &schema, trace_operations_table_name(table_name)),
90 ]);
91 }
92
93 PermissionTableTargets::resolved(targets)
94}
95
96#[async_trait]
97impl OpenTelemetryProtocolHandler for Instance {
98 #[tracing::instrument(skip_all)]
99 async fn metrics(
100 &self,
101 request: ExportMetricsServiceRequest,
102 ctx: QueryContextRef,
103 memory_limiter: Option<(ServerMemoryLimiter, u64)>,
104 ) -> ServerResult<MetricsIngestOutcome> {
105 self.plugins
106 .get::<PermissionCheckerRef>()
107 .as_ref()
108 .check_permission(ctx.current_user(), PermissionReq::Action(OTLP_WRITE))
109 .context(AuthSnafu)?;
110
111 let interceptor_ref = self
112 .plugins
113 .get::<OpenTelemetryProtocolInterceptorRef<servers::error::Error>>();
114 interceptor_ref.pre_execute(ctx.clone())?;
115 let ctx = Arc::new(ctx.fork());
116
117 let mut metric_ctx = ctx
118 .protocol_ctx()
119 .get_otlp_metric_ctx()
120 .cloned()
121 .unwrap_or_default();
122 metric_ctx.resource_info = self.otlp_resource_info;
123
124 let mut prepared = otlp::metrics::prepare_metrics(request, &mut metric_ctx)?;
125 let targets = {
127 let catalog = ctx.current_catalog();
128 let schema = ctx.current_schema();
129 let mut table_names = HashSet::new();
130 prepared
131 .requests
132 .iter()
133 .filter(|request| table_names.insert(request.table_name.as_str()))
134 .map(|request| PermissionTableTarget::new(catalog, &schema, &request.table_name))
135 .collect::<Vec<_>>()
136 };
137 self.check_table_permission(
138 &ctx,
139 PermissionReq::Action(OTLP_WRITE),
140 PermissionTableTargets::resolved(targets),
141 )
142 .context(AuthSnafu)?;
143
144 let preflight = self
145 .inserter
146 .validate_and_normalize_otlp_metrics(
147 &mut prepared.requests,
148 &ctx,
149 metric_ctx.with_metric_engine,
150 ctx.extension(PHYSICAL_TABLE_PARAM)
151 .unwrap_or(GREPTIME_PHYSICAL_TABLE),
152 )
153 .await
154 .map_err(BoxedError::new)
155 .context(error::ExecuteGrpcQuerySnafu)?;
156 let rejections = prepared.point_rejections(preflight.rejections)?;
157 let otlp::metrics::MetricsConversion {
158 requests,
159 rows,
160 mut exemplars,
161 semantic_index,
162 resource_info,
163 mut outcome,
164 } = prepared.finish(rejections, &preflight.physical_tables)?;
165 if outcome.rejected_data_points > 0 {
166 warn!(
167 "Rejected {} OTLP metrics data points: {}",
168 outcome.rejected_data_points,
169 outcome.error_message.as_deref().unwrap_or_default()
170 );
171 }
172 if outcome.accepted_data_points == 0 {
173 return Ok(outcome);
174 }
175
176 let before = exemplars.len();
178 exemplars.retain(|exemplar| {
179 preflight
180 .physical_tables
181 .contains_key(exemplar.metric.as_ref())
182 });
183 servers::metrics::EXEMPLARS_DROPPED_TOTAL
184 .with_label_values(&["otlp"])
185 .inc_by((before - exemplars.len()) as u64);
186 if !exemplars.is_empty() {
187 self.check_table_permission(
188 &ctx,
189 PermissionReq::Action(OTLP_WRITE),
190 PermissionTableTargets::resolved(vec![PermissionTableTarget::new(
191 ctx.current_catalog(),
192 ctx.current_schema(),
193 METRICS_EXEMPLARS_TABLE,
194 )]),
195 )
196 .context(AuthSnafu)?;
197 }
198
199 let ctx = if !exemplars.is_empty()
200 && let Some((limiter, held_bytes)) = memory_limiter
201 {
202 let bytes =
203 servers::exemplar::estimated_row_bytes(std::iter::once(exemplars.as_slice()))
204 .and_then(|bytes| u64::try_from(bytes).ok())
205 .ok_or_else(|| {
206 error::MemoryLimitExceededSnafu.into_error(
207 common_memory_manager::Error::MemoryLimitExceeded {
208 requested_bytes: u64::MAX,
209 limit_bytes: limiter.limit_bytes(),
210 },
211 )
212 })?;
213 limiter.check_request_fits(bytes, held_bytes)?;
214 let reservation = Arc::new(if held_bytes > 0 {
216 limiter.try_acquire(bytes).ok_or_else(|| {
217 error::MemoryLimitExceededSnafu.into_error(
218 common_memory_manager::Error::MemoryLimitExceeded {
219 requested_bytes: bytes,
220 limit_bytes: limiter.limit_bytes(),
221 },
222 )
223 })?
224 } else {
225 limiter.acquire(bytes).await?
226 });
227 let mut ctx = ctx.fork();
228 ctx.set_write_progress(Arc::new(WriteProgress::with_reservation(
230 reservation.granted_bytes(),
231 reservation,
232 )));
233 Arc::new(ctx)
234 } else {
235 ctx
236 };
237 let exemplar_rows = exemplars.len();
238 let (exemplars, _) = servers::exemplar::into_rows(exemplars)?;
239 let total_rows = rows.checked_add(exemplar_rows).ok_or_else(|| {
240 error::InvalidOtlpMetricInputSnafu {
241 reason: "combined metric and exemplar row count overflow",
242 }
243 .build()
244 })?;
245 let ctx = admit_write(total_rows as u64, &ctx)
246 .await
247 .map_err(BoxedError::new)
248 .context(error::ExecuteGrpcQuerySnafu)?;
249 OTLP_METRICS_ROWS.inc_by(rows as u64);
250
251 let ctx = {
252 let mut c = (*ctx).clone();
253 c.set_extension(SEMANTIC_SIGNAL_TYPE, SIGNAL_TYPE_METRIC);
254 c.set_extension(SEMANTIC_SOURCE, SOURCE_OPENTELEMETRY);
255 if let Some(index) = semantic_index.encode(&c.current_schema()) {
259 c.set_extension(SEMANTIC_PER_TABLE_INDEX_KEY, index);
260 }
261 c.set_extension(OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM.to_string());
262 Arc::new(c)
263 };
264
265 if exemplar_rows > 0 {
266 let confirmed = Arc::new(ctx.with_confirmed_writes());
267 let source_names = exemplars
268 .inserts
269 .iter()
270 .filter_map(|request| request.rows.as_ref())
271 .flat_map(|rows| {
272 let index = rows
273 .schema
274 .iter()
275 .position(|c| c.column_name == EXEMPLAR_METRIC_COLUMN);
276 rows.rows.iter().filter_map(move |row| {
277 match &row.values.get(index?)?.value_data {
278 Some(ValueData::StringValue(name)) => Some(name.as_str()),
279 _ => None,
280 }
281 })
282 })
283 .collect::<HashSet<_>>();
284 let (source_requests, other): (Vec<_>, Vec<_>) = requests
285 .inserts
286 .into_iter()
287 .partition(|request| source_names.contains(request.table_name.as_str()));
288 let prepared = self
289 .inserter
290 .prepare_metric_row_inserts(
291 RowInsertRequests {
292 inserts: source_requests,
293 },
294 confirmed.clone(),
295 &self.statement_executor,
296 ctx.extension(PHYSICAL_TABLE_PARAM)
297 .unwrap_or(GREPTIME_PHYSICAL_TABLE)
298 .to_string(),
299 )
300 .await
301 .map_err(BoxedError::new)
302 .context(error::ExecuteGrpcQuerySnafu);
303 let mut exemplar_output = Ok(Output::new_with_affected_rows(0));
304 let (metric_progress, other_output) = tokio::join!(
305 async {
306 let prepared = prepared?;
307 let sources = prepared.source_tables().cloned().collect::<Vec<_>>();
308 let output = self
309 .inserter
310 .handle_prepared_metric_inserts(prepared, confirmed.clone())
311 .await
312 .map_err(BoxedError::new)
313 .context(error::ExecuteGrpcQuerySnafu)?;
314 outcome.write_cost += output.meta.cost;
315 let progress = output.write_progress().await;
317 if progress.error.is_none() {
318 exemplar_output = self
319 .inserter
320 .handle_exemplar_inserts(
321 exemplars,
322 confirmed.clone(),
323 &self.statement_executor,
324 &sources,
325 )
326 .await
327 .map_err(BoxedError::new)
328 .context(error::ExecuteGrpcQuerySnafu);
329 }
330 Ok::<_, error::Error>(progress)
331 },
332 async {
333 if other.is_empty() {
334 Ok(Output::new_with_affected_rows(0))
335 } else {
336 self.handle_otlp_metric_row_inserts(
337 RowInsertRequests { inserts: other },
338 confirmed.clone(),
339 &metric_ctx,
340 )
341 .await
342 }
343 },
344 );
345 let mut progress = match metric_progress {
346 Ok(progress) => progress,
347 Err(error) => common_query::WriteProgress::failed(BoxedError::new(error)),
348 };
349 for output in [other_output, exemplar_output] {
350 match output {
351 Ok(output) => {
352 outcome.write_cost += output.meta.cost;
353 progress.merge(output.write_progress().await);
354 }
355 Err(error) => {
356 progress.merge(common_query::WriteProgress::failed(BoxedError::new(error)))
357 }
358 }
359 }
360 progress
361 .into_result()
362 .context(error::ExecuteGrpcQuerySnafu)?;
363 } else {
364 let output = self
365 .handle_otlp_metric_row_inserts(requests, ctx.clone(), &metric_ctx)
366 .await?;
367 outcome.write_cost = output.meta.cost;
368 }
369
370 if let Some(resource_info) = resource_info {
374 let written = match self.check_row_insert_permission(
375 &resource_info,
376 &ctx,
377 PermissionReq::Action(OTLP_WRITE),
378 ) {
379 Ok(_) => self
380 .handle_row_inserts(resource_info, ctx, false, false)
381 .await
382 .map_err(BoxedError::new)
383 .map_err(|e| e.to_string()),
384 Err(e) => Err(e.to_string()),
385 };
386 match written {
387 Ok(descriptor_output) => outcome.write_cost += descriptor_output.meta.cost,
388 Err(e) => {
389 OTLP_RESOURCE_INFO_WRITE_ERRORS.inc();
390 warn!("Failed to write the OTLP resource descriptor table: {e}");
391 outcome.error_message.get_or_insert(format!(
392 "metric data was accepted, but writing the resource \
393 descriptor table `{}` failed: {e}",
394 otlp::metrics::OTEL_RESOURCE_INFO_TABLE_NAME
395 ));
396 }
397 }
398 }
399
400 Ok(outcome)
401 }
402
403 #[tracing::instrument(skip_all)]
404 async fn traces(
405 &self,
406 _pipeline_handler: PipelineHandlerRef,
407 request: ExportTraceServiceRequest,
408 pipeline: PipelineWay,
409 _pipeline_params: GreptimePipelineParams,
410 table_name: String,
411 ctx: QueryContextRef,
412 ) -> ServerResult<TraceIngestOutcome> {
413 self.plugins
414 .get::<PermissionCheckerRef>()
415 .as_ref()
416 .check_permission(ctx.current_user(), PermissionReq::Action(OTLP_WRITE))
417 .context(AuthSnafu)?;
418
419 let interceptor_ref = self
420 .plugins
421 .get::<OpenTelemetryProtocolInterceptorRef<servers::error::Error>>();
422 interceptor_ref.pre_execute(ctx.clone())?;
423 let ctx = Arc::new(ctx.fork());
424
425 let conventions = trace_conventions(&request);
427 let spans = otlp::trace::span::parse(request);
428 let targets = trace_permission_targets(&table_name, &spans, &ctx);
429 self.check_table_permission(&ctx, PermissionReq::Action(OTLP_WRITE), targets)
430 .context(AuthSnafu)?;
431 let rows = spans.iter().map(|group| group.spans.len() as u64).sum();
433 let ctx = admit_write(rows, &ctx)
434 .await
435 .map_err(BoxedError::new)
436 .context(error::ExecuteGrpcQuerySnafu)?;
437 self.ingest_trace_spans(&pipeline, table_name, spans, &conventions, ctx)
438 .await
439 }
440
441 #[tracing::instrument(skip_all)]
442 async fn logs(
443 &self,
444 pipeline_handler: PipelineHandlerRef,
445 request: ExportLogsServiceRequest,
446 pipeline: PipelineWay,
447 pipeline_params: GreptimePipelineParams,
448 table_name: String,
449 ctx: QueryContextRef,
450 ) -> ServerResult<Vec<Output>> {
451 self.plugins
452 .get::<PermissionCheckerRef>()
453 .as_ref()
454 .check_permission(ctx.current_user(), PermissionReq::Action(OTLP_WRITE))
455 .context(AuthSnafu)?;
456
457 let interceptor_ref = self
458 .plugins
459 .get::<OpenTelemetryProtocolInterceptorRef<servers::error::Error>>();
460 interceptor_ref.pre_execute(ctx.clone())?;
461 let ctx = Arc::new(ctx.fork());
462
463 let ctx = {
466 let mut c = (*ctx).clone();
467 c.set_extension(SEMANTIC_SIGNAL_TYPE, SIGNAL_TYPE_LOG);
468 c.set_extension(SEMANTIC_SOURCE, SOURCE_OPENTELEMETRY);
469 if matches!(pipeline, PipelineWay::OtlpLogDirect(_)) {
473 c.set_extension(TABLE_DATA_MODEL, TABLE_DATA_MODEL_LOG_V1);
474 c.set_extension(SEMANTIC_PIPELINE, TABLE_DATA_MODEL_LOG_V1);
475 }
478 Arc::new(c)
479 };
480
481 let opt_req = otlp::logs::to_grpc_insert_requests(
482 request,
483 pipeline,
484 pipeline_params,
485 table_name,
486 &ctx,
487 pipeline_handler,
488 )
489 .await?;
490
491 let mut batches = opt_req.as_req_iter(ctx).collect::<Vec<_>>();
492 for (temp_ctx, requests) in &batches {
493 self.check_row_insert_permission(requests, temp_ctx, PermissionReq::Action(OTLP_WRITE))
494 .context(AuthSnafu)?;
495 }
496 admit_row_insert_batches(&mut batches)
497 .await
498 .map_err(BoxedError::new)
499 .context(error::ExecuteGrpcQuerySnafu)?;
500
501 let mut outputs = Vec::with_capacity(batches.len());
502 for (temp_ctx, requests) in batches {
503 let cnt = requests
504 .inserts
505 .iter()
506 .filter_map(|r| r.rows.as_ref().map(|r| r.rows.len()))
507 .sum::<usize>();
508
509 let o = self
510 .handle_log_inserts(requests, temp_ctx)
511 .await
512 .inspect(|_| OTLP_LOGS_ROWS.inc_by(cnt as u64))
513 .map_err(BoxedError::new)
514 .context(error::ExecuteGrpcQuerySnafu)?;
515 outputs.push(o);
516 }
517
518 Ok(outputs)
519 }
520}
521
522impl Instance {
523 pub async fn handle_otlp_metric_row_inserts(
526 &self,
527 requests: RowInsertRequests,
528 ctx: QueryContextRef,
529 metric_ctx: &OtlpMetricCtx,
530 ) -> ServerResult<Output> {
531 let physical_table = ctx
532 .extension(PHYSICAL_TABLE_PARAM)
533 .unwrap_or(GREPTIME_PHYSICAL_TABLE)
534 .to_string();
535 let batcher = self
539 .logical_batcher()
540 .filter(|_| ctx.logical_batching_enabled() && metric_ctx.with_metric_engine);
541 let batcher = if batcher.is_some()
542 && self
543 .inserter
544 .can_batch_metric_rows(&requests, &ctx, &physical_table)
545 .await
546 .map_err(BoxedError::new)
547 .context(error::ExecuteGrpcQuerySnafu)?
548 {
549 batcher
550 } else {
551 None
552 };
553
554 if let Some(batcher) = batcher {
556 let (mut output, cost) = batcher
557 .submit_output_with(requests, ctx.clone(), |mut requests| {
558 let ctx = ctx.clone();
559 async move {
560 Inserter::meter_row_inserts(&mut requests, &ctx)
561 .await
562 .map_err(BoxedError::new)
563 .context(error::ExecuteGrpcQuerySnafu)
564 }
565 })
566 .await?;
567 output.meta.cost = cost as _;
568 Ok(output)
569 } else if !metric_ctx.with_metric_engine {
570 self.handle_row_inserts(requests, ctx.clone(), false, true)
571 .await
572 .map_err(BoxedError::new)
573 .context(error::ExecuteGrpcQuerySnafu)
574 } else {
575 self.handle_metric_row_inserts(requests, ctx.clone(), physical_table)
576 .await
577 .map_err(BoxedError::new)
578 .context(error::ExecuteGrpcQuerySnafu)
579 }
580 }
581}
582
583#[cfg(test)]
584mod tests {
585 use std::sync::Arc;
586
587 use auth::{PermissionTableTarget, PermissionTableTargets};
588 use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
589 use opentelemetry_proto::tonic::common::v1::{AnyValue, KeyValue, any_value};
590 use opentelemetry_proto::tonic::resource::v1::Resource;
591 use opentelemetry_proto::tonic::trace::v1::{ResourceSpans, ScopeSpans, Span};
592 use session::context::QueryContext;
593
594 use super::trace_permission_targets;
595
596 #[test]
597 fn test_trace_permission_targets() {
598 let request = ExportTraceServiceRequest {
599 resource_spans: vec![ResourceSpans {
600 resource: Some(Resource {
601 attributes: vec![KeyValue {
602 key: "service.name".to_string(),
603 value: Some(AnyValue {
604 value: Some(any_value::Value::StringValue("frontend".to_string())),
605 }),
606 ..Default::default()
607 }],
608 ..Default::default()
609 }),
610 scope_spans: vec![ScopeSpans {
611 spans: vec![Span::default()],
612 ..Default::default()
613 }],
614 ..Default::default()
615 }],
616 };
617 let groups = servers::otlp::trace::span::parse(request);
618 let ctx = Arc::new(QueryContext::with("greptime", "public"));
619
620 assert_eq!(
621 PermissionTableTargets::Resolved(vec![
622 PermissionTableTarget::new("greptime", "public", "traces"),
623 PermissionTableTarget::new("greptime", "public", "traces_services"),
624 PermissionTableTarget::new("greptime", "public", "traces_operations"),
625 ]),
626 trace_permission_targets("traces", &groups, &ctx)
627 );
628 assert_eq!(
629 PermissionTableTargets::Unresolved,
630 trace_permission_targets("", &groups, &ctx)
631 );
632 }
633}