Skip to main content

frontend/instance/
otlp.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15mod trace_ingest;
16pub mod trace_semconv;
17pub mod trace_types;
18
19use std::collections::HashSet;
20use std::sync::Arc;
21
22use api::v1::RowInsertRequests;
23use api::v1::value::ValueData;
24use async_trait::async_trait;
25use auth::{
26    OTLP_WRITE, PermissionChecker, PermissionCheckerRef, PermissionReq, PermissionTableTarget,
27    PermissionTableTargets,
28};
29use client::Output;
30use common_catalog::consts::{
31    METRICS_EXEMPLARS_TABLE, trace_operations_table_name, trace_services_table_name,
32};
33use common_error::ext::BoxedError;
34use common_query::prelude::{EXEMPLAR_METRIC_COLUMN, GREPTIME_PHYSICAL_TABLE};
35use common_telemetry::{tracing, warn};
36use opentelemetry_proto::tonic::collector::logs::v1::ExportLogsServiceRequest;
37use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
38use operator::insert::{Inserter, admit_row_insert_batches, admit_write};
39use otel_arrow_rust::proto::opentelemetry::collector::metrics::v1::ExportMetricsServiceRequest;
40use pipeline::{GreptimePipelineParams, PipelineWay};
41use servers::error::{self, AuthSnafu, Result as ServerResult};
42use servers::http::prom_store::PHYSICAL_TABLE_PARAM;
43use servers::interceptor::{OpenTelemetryProtocolInterceptor, OpenTelemetryProtocolInterceptorRef};
44use servers::otlp;
45use servers::otlp::trace::span::TraceSpanGroup;
46use servers::query_handler::{
47    MetricsIngestOutcome, OpenTelemetryProtocolHandler, PipelineHandlerRef, TraceIngestOutcome,
48};
49use servers::request_memory_limiter::ServerMemoryLimiter;
50use session::context::QueryContextRef;
51use session::protocol_ctx::OtlpMetricCtx;
52use session::write_progress::WriteProgress;
53use snafu::{IntoError, ResultExt};
54use table::requests::{
55    OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM, SEMANTIC_PER_TABLE_INDEX_KEY,
56    SEMANTIC_PIPELINE, SEMANTIC_SIGNAL_TYPE, SEMANTIC_SOURCE, SIGNAL_TYPE_LOG, SIGNAL_TYPE_METRIC,
57    SOURCE_OPENTELEMETRY, TABLE_DATA_MODEL, TABLE_DATA_MODEL_LOG_V1,
58};
59
60pub(crate) use self::trace_ingest::TraceAuxCache;
61use self::trace_ingest::trace_conventions;
62use crate::instance::Instance;
63use crate::metrics::{OTLP_LOGS_ROWS, OTLP_METRICS_ROWS, OTLP_RESOURCE_INFO_WRITE_ERRORS};
64
65fn trace_permission_targets(
66    table_name: &str,
67    groups: &[TraceSpanGroup],
68    ctx: &QueryContextRef,
69) -> PermissionTableTargets {
70    let catalog = ctx.current_catalog();
71    let schema = ctx.current_schema();
72    if catalog.is_empty() || schema.is_empty() || table_name.is_empty() {
73        return PermissionTableTargets::Unresolved;
74    }
75
76    let has_spans = groups.iter().any(|group| !group.spans.is_empty());
77    if !has_spans {
78        return PermissionTableTargets::resolved(Vec::new());
79    }
80
81    let mut targets = vec![PermissionTableTarget::new(catalog, &schema, table_name)];
82    if groups
83        .iter()
84        .flat_map(|group| &group.spans)
85        .any(|span| span.service_name.is_some())
86    {
87        targets.extend([
88            PermissionTableTarget::new(catalog, &schema, trace_services_table_name(table_name)),
89            PermissionTableTarget::new(catalog, &schema, trace_operations_table_name(table_name)),
90        ]);
91    }
92
93    PermissionTableTargets::resolved(targets)
94}
95
96#[async_trait]
97impl OpenTelemetryProtocolHandler for Instance {
98    #[tracing::instrument(skip_all)]
99    async fn metrics(
100        &self,
101        request: ExportMetricsServiceRequest,
102        ctx: QueryContextRef,
103        memory_limiter: Option<(ServerMemoryLimiter, u64)>,
104    ) -> ServerResult<MetricsIngestOutcome> {
105        self.plugins
106            .get::<PermissionCheckerRef>()
107            .as_ref()
108            .check_permission(ctx.current_user(), PermissionReq::Action(OTLP_WRITE))
109            .context(AuthSnafu)?;
110
111        let interceptor_ref = self
112            .plugins
113            .get::<OpenTelemetryProtocolInterceptorRef<servers::error::Error>>();
114        interceptor_ref.pre_execute(ctx.clone())?;
115        let ctx = Arc::new(ctx.fork());
116
117        let mut metric_ctx = ctx
118            .protocol_ctx()
119            .get_otlp_metric_ctx()
120            .cloned()
121            .unwrap_or_default();
122        metric_ctx.resource_info = self.otlp_resource_info;
123
124        let mut prepared = otlp::metrics::prepare_metrics(request, &mut metric_ctx)?;
125        // Authorize source tables before preflight can expose schemas or discard points.
126        let targets = {
127            let catalog = ctx.current_catalog();
128            let schema = ctx.current_schema();
129            let mut table_names = HashSet::new();
130            prepared
131                .requests
132                .iter()
133                .filter(|request| table_names.insert(request.table_name.as_str()))
134                .map(|request| PermissionTableTarget::new(catalog, &schema, &request.table_name))
135                .collect::<Vec<_>>()
136        };
137        self.check_table_permission(
138            &ctx,
139            PermissionReq::Action(OTLP_WRITE),
140            PermissionTableTargets::resolved(targets),
141        )
142        .context(AuthSnafu)?;
143
144        let preflight = self
145            .inserter
146            .validate_and_normalize_otlp_metrics(
147                &mut prepared.requests,
148                &ctx,
149                metric_ctx.with_metric_engine,
150                ctx.extension(PHYSICAL_TABLE_PARAM)
151                    .unwrap_or(GREPTIME_PHYSICAL_TABLE),
152            )
153            .await
154            .map_err(BoxedError::new)
155            .context(error::ExecuteGrpcQuerySnafu)?;
156        let rejections = prepared.point_rejections(preflight.rejections)?;
157        let otlp::metrics::MetricsConversion {
158            requests,
159            rows,
160            mut exemplars,
161            semantic_index,
162            resource_info,
163            mut outcome,
164        } = prepared.finish(rejections, &preflight.physical_tables)?;
165        if outcome.rejected_data_points > 0 {
166            warn!(
167                "Rejected {} OTLP metrics data points: {}",
168                outcome.rejected_data_points,
169                outcome.error_message.as_deref().unwrap_or_default()
170            );
171        }
172        if outcome.accepted_data_points == 0 {
173            return Ok(outcome);
174        }
175
176        // The engine option only controls new tables; existing ordinary sources cannot bind exemplars.
177        let before = exemplars.len();
178        exemplars.retain(|exemplar| {
179            preflight
180                .physical_tables
181                .contains_key(exemplar.metric.as_ref())
182        });
183        servers::metrics::EXEMPLARS_DROPPED_TOTAL
184            .with_label_values(&["otlp"])
185            .inc_by((before - exemplars.len()) as u64);
186        if !exemplars.is_empty() {
187            self.check_table_permission(
188                &ctx,
189                PermissionReq::Action(OTLP_WRITE),
190                PermissionTableTargets::resolved(vec![PermissionTableTarget::new(
191                    ctx.current_catalog(),
192                    ctx.current_schema(),
193                    METRICS_EXEMPLARS_TABLE,
194                )]),
195            )
196            .context(AuthSnafu)?;
197        }
198
199        let ctx = if !exemplars.is_empty()
200            && let Some((limiter, held_bytes)) = memory_limiter
201        {
202            let bytes =
203                servers::exemplar::estimated_row_bytes(std::iter::once(exemplars.as_slice()))
204                    .and_then(|bytes| u64::try_from(bytes).ok())
205                    .ok_or_else(|| {
206                        error::MemoryLimitExceededSnafu.into_error(
207                            common_memory_manager::Error::MemoryLimitExceeded {
208                                requested_bytes: u64::MAX,
209                                limit_bytes: limiter.limit_bytes(),
210                            },
211                        )
212                    })?;
213            limiter.check_request_fits(bytes, held_bytes)?;
214            // Holding body permits while waiting for expansion can stall concurrent requests.
215            let reservation = Arc::new(if held_bytes > 0 {
216                limiter.try_acquire(bytes).ok_or_else(|| {
217                    error::MemoryLimitExceededSnafu.into_error(
218                        common_memory_manager::Error::MemoryLimitExceeded {
219                            requested_bytes: bytes,
220                            limit_bytes: limiter.limit_bytes(),
221                        },
222                    )
223                })?
224            } else {
225                limiter.acquire(bytes).await?
226            });
227            let mut ctx = ctx.fork();
228            // Queued and detached writes retain the reservation after caller cancellation.
229            ctx.set_write_progress(Arc::new(WriteProgress::with_reservation(
230                reservation.granted_bytes(),
231                reservation,
232            )));
233            Arc::new(ctx)
234        } else {
235            ctx
236        };
237        let exemplar_rows = exemplars.len();
238        let (exemplars, _) = servers::exemplar::into_rows(exemplars)?;
239        let total_rows = rows.checked_add(exemplar_rows).ok_or_else(|| {
240            error::InvalidOtlpMetricInputSnafu {
241                reason: "combined metric and exemplar row count overflow",
242            }
243            .build()
244        })?;
245        let ctx = admit_write(total_rows as u64, &ctx)
246            .await
247            .map_err(BoxedError::new)
248            .context(error::ExecuteGrpcQuerySnafu)?;
249        OTLP_METRICS_ROWS.inc_by(rows as u64);
250
251        let ctx = {
252            let mut c = (*ctx).clone();
253            c.set_extension(SEMANTIC_SIGNAL_TYPE, SIGNAL_TYPE_METRIC);
254            c.set_extension(SEMANTIC_SOURCE, SOURCE_OPENTELEMETRY);
255            // Per-table metric specifics + resource/scope lineage ride this
256            // internal channel; the auto-create path folds them per schema and
257            // table name.
258            if let Some(index) = semantic_index.encode(&c.current_schema()) {
259                c.set_extension(SEMANTIC_PER_TABLE_INDEX_KEY, index);
260            }
261            c.set_extension(OTLP_METRIC_COMPAT_KEY, OTLP_METRIC_COMPAT_PROM.to_string());
262            Arc::new(c)
263        };
264
265        if exemplar_rows > 0 {
266            let confirmed = Arc::new(ctx.with_confirmed_writes());
267            let source_names = exemplars
268                .inserts
269                .iter()
270                .filter_map(|request| request.rows.as_ref())
271                .flat_map(|rows| {
272                    let index = rows
273                        .schema
274                        .iter()
275                        .position(|c| c.column_name == EXEMPLAR_METRIC_COLUMN);
276                    rows.rows.iter().filter_map(move |row| {
277                        match &row.values.get(index?)?.value_data {
278                            Some(ValueData::StringValue(name)) => Some(name.as_str()),
279                            _ => None,
280                        }
281                    })
282                })
283                .collect::<HashSet<_>>();
284            let (source_requests, other): (Vec<_>, Vec<_>) = requests
285                .inserts
286                .into_iter()
287                .partition(|request| source_names.contains(request.table_name.as_str()));
288            let prepared = self
289                .inserter
290                .prepare_metric_row_inserts(
291                    RowInsertRequests {
292                        inserts: source_requests,
293                    },
294                    confirmed.clone(),
295                    &self.statement_executor,
296                    ctx.extension(PHYSICAL_TABLE_PARAM)
297                        .unwrap_or(GREPTIME_PHYSICAL_TABLE)
298                        .to_string(),
299                )
300                .await
301                .map_err(BoxedError::new)
302                .context(error::ExecuteGrpcQuerySnafu);
303            let mut exemplar_output = Ok(Output::new_with_affected_rows(0));
304            let (metric_progress, other_output) = tokio::join!(
305                async {
306                    let prepared = prepared?;
307                    let sources = prepared.source_tables().cloned().collect::<Vec<_>>();
308                    let output = self
309                        .inserter
310                        .handle_prepared_metric_inserts(prepared, confirmed.clone())
311                        .await
312                        .map_err(BoxedError::new)
313                        .context(error::ExecuteGrpcQuerySnafu)?;
314                    outcome.write_cost += output.meta.cost;
315                    // Successful insertion outputs can still carry storage failures.
316                    let progress = output.write_progress().await;
317                    if progress.error.is_none() {
318                        exemplar_output = self
319                            .inserter
320                            .handle_exemplar_inserts(
321                                exemplars,
322                                confirmed.clone(),
323                                &self.statement_executor,
324                                &sources,
325                            )
326                            .await
327                            .map_err(BoxedError::new)
328                            .context(error::ExecuteGrpcQuerySnafu);
329                    }
330                    Ok::<_, error::Error>(progress)
331                },
332                async {
333                    if other.is_empty() {
334                        Ok(Output::new_with_affected_rows(0))
335                    } else {
336                        self.handle_otlp_metric_row_inserts(
337                            RowInsertRequests { inserts: other },
338                            confirmed.clone(),
339                            &metric_ctx,
340                        )
341                        .await
342                    }
343                },
344            );
345            let mut progress = match metric_progress {
346                Ok(progress) => progress,
347                Err(error) => common_query::WriteProgress::failed(BoxedError::new(error)),
348            };
349            for output in [other_output, exemplar_output] {
350                match output {
351                    Ok(output) => {
352                        outcome.write_cost += output.meta.cost;
353                        progress.merge(output.write_progress().await);
354                    }
355                    Err(error) => {
356                        progress.merge(common_query::WriteProgress::failed(BoxedError::new(error)))
357                    }
358                }
359            }
360            progress
361                .into_result()
362                .context(error::ExecuteGrpcQuerySnafu)?;
363        } else {
364            let output = self
365                .handle_otlp_metric_row_inserts(requests, ctx.clone(), &metric_ctx)
366                .await?;
367            outcome.write_cost = output.meta.cost;
368        }
369
370        // Derived enrichment follows the accepted metric submission, which may
371        // still be queued in asynchronous mode. Failures remain warning-only
372        // to avoid retrying metric data the server already accepted.
373        if let Some(resource_info) = resource_info {
374            let written = match self.check_row_insert_permission(
375                &resource_info,
376                &ctx,
377                PermissionReq::Action(OTLP_WRITE),
378            ) {
379                Ok(_) => self
380                    .handle_row_inserts(resource_info, ctx, false, false)
381                    .await
382                    .map_err(BoxedError::new)
383                    .map_err(|e| e.to_string()),
384                Err(e) => Err(e.to_string()),
385            };
386            match written {
387                Ok(descriptor_output) => outcome.write_cost += descriptor_output.meta.cost,
388                Err(e) => {
389                    OTLP_RESOURCE_INFO_WRITE_ERRORS.inc();
390                    warn!("Failed to write the OTLP resource descriptor table: {e}");
391                    outcome.error_message.get_or_insert(format!(
392                        "metric data was accepted, but writing the resource \
393                         descriptor table `{}` failed: {e}",
394                        otlp::metrics::OTEL_RESOURCE_INFO_TABLE_NAME
395                    ));
396                }
397            }
398        }
399
400        Ok(outcome)
401    }
402
403    #[tracing::instrument(skip_all)]
404    async fn traces(
405        &self,
406        _pipeline_handler: PipelineHandlerRef,
407        request: ExportTraceServiceRequest,
408        pipeline: PipelineWay,
409        _pipeline_params: GreptimePipelineParams,
410        table_name: String,
411        ctx: QueryContextRef,
412    ) -> ServerResult<TraceIngestOutcome> {
413        self.plugins
414            .get::<PermissionCheckerRef>()
415            .as_ref()
416            .check_permission(ctx.current_user(), PermissionReq::Action(OTLP_WRITE))
417            .context(AuthSnafu)?;
418
419        let interceptor_ref = self
420            .plugins
421            .get::<OpenTelemetryProtocolInterceptorRef<servers::error::Error>>();
422        interceptor_ref.pre_execute(ctx.clone())?;
423        let ctx = Arc::new(ctx.fork());
424
425        // `schema_url` is consumed by `parse`, so derive conventions first.
426        let conventions = trace_conventions(&request);
427        let spans = otlp::trace::span::parse(request);
428        let targets = trace_permission_targets(&table_name, &spans, &ctx);
429        self.check_table_permission(&ctx, PermissionReq::Action(OTLP_WRITE), targets)
430            .context(AuthSnafu)?;
431        // Count the external spans once, before chunking, retries, or derived tables.
432        let rows = spans.iter().map(|group| group.spans.len() as u64).sum();
433        let ctx = admit_write(rows, &ctx)
434            .await
435            .map_err(BoxedError::new)
436            .context(error::ExecuteGrpcQuerySnafu)?;
437        self.ingest_trace_spans(&pipeline, table_name, spans, &conventions, ctx)
438            .await
439    }
440
441    #[tracing::instrument(skip_all)]
442    async fn logs(
443        &self,
444        pipeline_handler: PipelineHandlerRef,
445        request: ExportLogsServiceRequest,
446        pipeline: PipelineWay,
447        pipeline_params: GreptimePipelineParams,
448        table_name: String,
449        ctx: QueryContextRef,
450    ) -> ServerResult<Vec<Output>> {
451        self.plugins
452            .get::<PermissionCheckerRef>()
453            .as_ref()
454            .check_permission(ctx.current_user(), PermissionReq::Action(OTLP_WRITE))
455            .context(AuthSnafu)?;
456
457        let interceptor_ref = self
458            .plugins
459            .get::<OpenTelemetryProtocolInterceptorRef<servers::error::Error>>();
460        interceptor_ref.pre_execute(ctx.clone())?;
461        let ctx = Arc::new(ctx.fork());
462
463        // `as_req_iter` clones this ctx into each `temp_ctx`, so identity set here
464        // reaches the context that drives table auto-create.
465        let ctx = {
466            let mut c = (*ctx).clone();
467            c.set_extension(SEMANTIC_SIGNAL_TYPE, SIGNAL_TYPE_LOG);
468            c.set_extension(SEMANTIC_SOURCE, SOURCE_OPENTELEMETRY);
469            // Only applied when the table is auto-created. The v1 rows are built only
470            // when the table didn't exist at read time; pipeline-defined tables have no
471            // built-in schema.
472            if matches!(pipeline, PipelineWay::OtlpLogDirect(_)) {
473                c.set_extension(TABLE_DATA_MODEL, TABLE_DATA_MODEL_LOG_V1);
474                c.set_extension(SEMANTIC_PIPELINE, TABLE_DATA_MODEL_LOG_V1);
475                // TODO: declare the `service` entity on `service_name` once the cost of
476                // scanning log tables during entity-graph derivation is measured.
477            }
478            Arc::new(c)
479        };
480
481        let opt_req = otlp::logs::to_grpc_insert_requests(
482            request,
483            pipeline,
484            pipeline_params,
485            table_name,
486            &ctx,
487            pipeline_handler,
488        )
489        .await?;
490
491        let mut batches = opt_req.as_req_iter(ctx).collect::<Vec<_>>();
492        for (temp_ctx, requests) in &batches {
493            self.check_row_insert_permission(requests, temp_ctx, PermissionReq::Action(OTLP_WRITE))
494                .context(AuthSnafu)?;
495        }
496        admit_row_insert_batches(&mut batches)
497            .await
498            .map_err(BoxedError::new)
499            .context(error::ExecuteGrpcQuerySnafu)?;
500
501        let mut outputs = Vec::with_capacity(batches.len());
502        for (temp_ctx, requests) in batches {
503            let cnt = requests
504                .inserts
505                .iter()
506                .filter_map(|r| r.rows.as_ref().map(|r| r.rows.len()))
507                .sum::<usize>();
508
509            let o = self
510                .handle_log_inserts(requests, temp_ctx)
511                .await
512                .inspect(|_| OTLP_LOGS_ROWS.inc_by(cnt as u64))
513                .map_err(BoxedError::new)
514                .context(error::ExecuteGrpcQuerySnafu)?;
515            outputs.push(o);
516        }
517
518        Ok(outputs)
519    }
520}
521
522impl Instance {
523    /// Inserts converted OTLP metrics through the eligible batching or ordinary write path.
524    /// The caller must check table permissions and admit the complete request first.
525    pub async fn handle_otlp_metric_row_inserts(
526        &self,
527        requests: RowInsertRequests,
528        ctx: QueryContextRef,
529        metric_ctx: &OtlpMetricCtx,
530    ) -> ServerResult<Output> {
531        let physical_table = ctx
532            .extension(PHYSICAL_TABLE_PARAM)
533            .unwrap_or(GREPTIME_PHYSICAL_TABLE)
534            .to_string();
535        // The bulk path converts each request's time index unit to the
536        // destination table's unit during batch alignment, so no pre-gate
537        // alignment is needed here.
538        let batcher = self
539            .logical_batcher()
540            .filter(|_| ctx.logical_batching_enabled() && metric_ctx.with_metric_engine);
541        let batcher = if batcher.is_some()
542            && self
543                .inserter
544                .can_batch_metric_rows(&requests, &ctx, &physical_table)
545                .await
546                .map_err(BoxedError::new)
547                .context(error::ExecuteGrpcQuerySnafu)?
548        {
549            batcher
550        } else {
551            None
552        };
553
554        // OTLP tables have one sample field in both the ordinary and physical paths.
555        if let Some(batcher) = batcher {
556            let (mut output, cost) = batcher
557                .submit_output_with(requests, ctx.clone(), |mut requests| {
558                    let ctx = ctx.clone();
559                    async move {
560                        Inserter::meter_row_inserts(&mut requests, &ctx)
561                            .await
562                            .map_err(BoxedError::new)
563                            .context(error::ExecuteGrpcQuerySnafu)
564                    }
565                })
566                .await?;
567            output.meta.cost = cost as _;
568            Ok(output)
569        } else if !metric_ctx.with_metric_engine {
570            self.handle_row_inserts(requests, ctx.clone(), false, true)
571                .await
572                .map_err(BoxedError::new)
573                .context(error::ExecuteGrpcQuerySnafu)
574        } else {
575            self.handle_metric_row_inserts(requests, ctx.clone(), physical_table)
576                .await
577                .map_err(BoxedError::new)
578                .context(error::ExecuteGrpcQuerySnafu)
579        }
580    }
581}
582
583#[cfg(test)]
584mod tests {
585    use std::sync::Arc;
586
587    use auth::{PermissionTableTarget, PermissionTableTargets};
588    use opentelemetry_proto::tonic::collector::trace::v1::ExportTraceServiceRequest;
589    use opentelemetry_proto::tonic::common::v1::{AnyValue, KeyValue, any_value};
590    use opentelemetry_proto::tonic::resource::v1::Resource;
591    use opentelemetry_proto::tonic::trace::v1::{ResourceSpans, ScopeSpans, Span};
592    use session::context::QueryContext;
593
594    use super::trace_permission_targets;
595
596    #[test]
597    fn test_trace_permission_targets() {
598        let request = ExportTraceServiceRequest {
599            resource_spans: vec![ResourceSpans {
600                resource: Some(Resource {
601                    attributes: vec![KeyValue {
602                        key: "service.name".to_string(),
603                        value: Some(AnyValue {
604                            value: Some(any_value::Value::StringValue("frontend".to_string())),
605                        }),
606                        ..Default::default()
607                    }],
608                    ..Default::default()
609                }),
610                scope_spans: vec![ScopeSpans {
611                    spans: vec![Span::default()],
612                    ..Default::default()
613                }],
614                ..Default::default()
615            }],
616        };
617        let groups = servers::otlp::trace::span::parse(request);
618        let ctx = Arc::new(QueryContext::with("greptime", "public"));
619
620        assert_eq!(
621            PermissionTableTargets::Resolved(vec![
622                PermissionTableTarget::new("greptime", "public", "traces"),
623                PermissionTableTarget::new("greptime", "public", "traces_services"),
624                PermissionTableTarget::new("greptime", "public", "traces_operations"),
625            ]),
626            trace_permission_targets("traces", &groups, &ctx)
627        );
628        assert_eq!(
629            PermissionTableTargets::Unresolved,
630            trace_permission_targets("", &groups, &ctx)
631        );
632    }
633}