table/requests/
semantic.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15//! Table semantic layer vocabulary.
16//!
17//! A thin layer of semantic metadata attached to a table via `table_options`, so
18//! machine consumers (LLM agents, alert/dashboard builders, MCP servers, ETL) can
19//! align a table with the observability concept it stands for without guessing
20//! from column names. See `docs/rfcs/2026-05-28-table-semantic-layer.md`.
21//!
22//! The vocabulary is intentionally small: a key earns its place only when it
23//! records something a consumer cannot cheaply and reliably recover from the
24//! schema/data itself. Keys whose value is already in the metric name by
25//! convention, is a constant, or duplicates an existing column are deliberately
26//! omitted rather than stamped for completeness.
27//!
28//! All public keys share the [`SEMANTIC_PREFIX`] namespace and are string-valued.
29//! [`is_semantic_option_key`] gates them through
30//! [`crate::requests::validate_table_option`], so they are accepted both on the
31//! ingestion auto-create path and on explicit `CREATE TABLE ... WITH (...)` DDL.
32
33use datatypes::prelude::ConcreteDataType;
34
35/// Reserved prefix for every public semantic table-option key.
36pub const SEMANTIC_PREFIX: &str = "greptime.semantic.";
37
38/// Internal `QueryContext` extension key carrying the per-table semantic index
39/// (a `{table_name -> {semantic_key: value}}` JSON blob) from the ingestion
40/// encode path to the auto-create site. Deliberately OUTSIDE [`SEMANTIC_PREFIX`]
41/// so it is not a valid table option and never leaks into a table's options.
42pub const SEMANTIC_PER_TABLE_INDEX_KEY: &str = "greptime.internal.semantic.per_table_index";
43
44// ---- Common keys (all signals) ----
45
46/// Signal kind: one of [`SIGNAL_TYPE_TRACE`] / [`SIGNAL_TYPE_LOG`] /
47/// [`SIGNAL_TYPE_METRIC`] / [`SIGNAL_TYPE_EVENT`].
48pub const SEMANTIC_SIGNAL_TYPE: &str = "greptime.semantic.signal_type";
49/// Ingestion ecosystem, e.g. [`SOURCE_OPENTELEMETRY`] / [`SOURCE_PROMETHEUS`].
50pub const SEMANTIC_SOURCE: &str = "greptime.semantic.source";
51/// Source protocol version, e.g. Prometheus remote write `1.0` / `2.0`.
52pub const SEMANTIC_SOURCE_VERSION: &str = "greptime.semantic.source_version";
53/// Internal ingestion pipeline / data model, e.g. `greptime_trace_v1`. The
54/// signal-agnostic successor to the engine-specific `table_data_model` option.
55pub const SEMANTIC_PIPELINE: &str = "greptime.semantic.pipeline";
56
57// ---- Trace keys ----
58
59/// Semantic-conventions version the rows conform to (e.g. the OTel schema URL),
60/// or [`SEMANTIC_VALUE_UNKNOWN`] / [`SEMANTIC_VALUE_MIXED`] when not single-valued.
61pub const SEMANTIC_TRACE_CONVENTIONS: &str = "greptime.semantic.trace.conventions";
62
63// ---- Metric keys ----
64
65/// Instrument kind: `counter` / `gauge` / `histogram` / `summary` /
66/// `updown_counter` / `gauge_histogram` / `info` / `stateset`.
67pub const SEMANTIC_METRIC_TYPE: &str = "greptime.semantic.metric.type";
68/// UCUM unit, e.g. `s`, `By`, `{request}`. Discarded by the row encoders, so it
69/// is unrecoverable once ingested.
70pub const SEMANTIC_METRIC_UNIT: &str = "greptime.semantic.metric.unit";
71/// Catalog-level `cumulative` / `delta` / `mixed` description for OTLP metrics.
72/// Per-series query behavior is determined from stored row identity instead.
73pub const SEMANTIC_METRIC_TEMPORALITY: &str = "greptime.semantic.metric.temporality";
74/// [`METADATA_QUALITY_DECLARED`] when the protocol stated the type, or
75/// [`METADATA_QUALITY_INFERRED`] when guessed from a name suffix.
76pub const SEMANTIC_METRIC_METADATA_QUALITY: &str = "greptime.semantic.metric.metadata_quality";
77/// Pre-translation OTel name when the table name was Prometheus-ised; the key a
78/// consumer uses to look the metric up in the OTel semantic conventions.
79pub const SEMANTIC_METRIC_ORIGINAL_NAME: &str = "greptime.semantic.metric.original_name";
80
81// ---- Entity keys (open sub-namespace) ----
82
83/// Reserved prefix for the entity-identity sub-namespace:
84/// `greptime.semantic.entity.<type>.{id|descriptive|scope}`. Unlike the rest of
85/// the vocabulary (a closed whitelist), entity types are open-ended (`service`,
86/// `host`, `k8s.pod`, `process`, `agent`, custom, ...), so keys here are validated
87/// by prefix + shape rather than membership. See
88/// `docs/rfcs/2026-06-25-entity-relationships-and-graph-query.md`.
89pub const SEMANTIC_ENTITY_PREFIX: &str = "greptime.semantic.entity.";
90
91/// The well-known entity-identity key auto-stamped on OTLP trace tables; its value
92/// is the `service_name` tag column, declaring the logical `service` entity.
93pub const SEMANTIC_ENTITY_SERVICE_ID: &str = "greptime.semantic.entity.service.id";
94
95/// The role a set of columns plays for an entity: `id` (identifying
96/// attributes), `descriptive`, or `scope`. Columns may be tags or fields; DDL
97/// validation only requires that they exist and render as stable strings
98/// ([`has_stable_string_form`]).
99#[derive(Debug, Clone, Copy, PartialEq, Eq)]
100pub enum EntityRole {
101    /// Identifying columns. Their **order is part of the identity**: the
102    /// entity id is their values joined in the declared order, so two tables
103    /// naming the same entity must list them the same way (broad to narrow)
104    /// or they name two entities.
105    Id,
106    Descriptive,
107    Scope,
108}
109
110impl EntityRole {
111    fn parse(role: &str) -> Option<Self> {
112        match role {
113            "id" => Some(EntityRole::Id),
114            "descriptive" => Some(EntityRole::Descriptive),
115            "scope" => Some(EntityRole::Scope),
116            _ => None,
117        }
118    }
119}
120
121// ---- Value constants ----
122
123pub const SIGNAL_TYPE_TRACE: &str = "trace";
124pub const SIGNAL_TYPE_LOG: &str = "log";
125pub const SIGNAL_TYPE_METRIC: &str = "metric";
126pub const SIGNAL_TYPE_EVENT: &str = "event";
127
128pub const SOURCE_OPENTELEMETRY: &str = "opentelemetry";
129pub const SOURCE_PROMETHEUS: &str = "prometheus";
130pub const SOURCE_INFLUXDB: &str = "influxdb";
131pub const SOURCE_OPENTSDB: &str = "opentsdb";
132pub const SOURCE_LOKI: &str = "loki";
133pub const SOURCE_ELASTICSEARCH: &str = "elasticsearch";
134
135pub const METADATA_QUALITY_DECLARED: &str = "declared";
136pub const METADATA_QUALITY_INFERRED: &str = "inferred";
137
138pub const METRIC_TEMPORALITY_CUMULATIVE: &str = "cumulative";
139pub const METRIC_TEMPORALITY_DELTA: &str = "delta";
140
141/// Sentinel for a key that cannot be determined at stamp time.
142pub const SEMANTIC_VALUE_UNKNOWN: &str = "unknown";
143/// Sentinel for a single-valued key that saw conflicting sources.
144pub const SEMANTIC_VALUE_MIXED: &str = "mixed";
145
146/// Every recognised public semantic table-option key. The set is a closed
147/// whitelist: keys under [`SEMANTIC_PREFIX`] that are not listed here are rejected,
148/// so an unknown key like `greptime.semantic.unknown_key` does not silently land
149/// in a table's options. Adding a key to the vocabulary means adding it here.
150pub const SEMANTIC_OPTION_KEYS: &[&str] = &[
151    SEMANTIC_SIGNAL_TYPE,
152    SEMANTIC_SOURCE,
153    SEMANTIC_SOURCE_VERSION,
154    SEMANTIC_PIPELINE,
155    SEMANTIC_TRACE_CONVENTIONS,
156    SEMANTIC_METRIC_TYPE,
157    SEMANTIC_METRIC_UNIT,
158    SEMANTIC_METRIC_TEMPORALITY,
159    SEMANTIC_METRIC_METADATA_QUALITY,
160    SEMANTIC_METRIC_ORIGINAL_NAME,
161];
162
163/// Returns true if `ty` is a syntactically valid entity type, e.g. `service`,
164/// `host`, `k8s.pod`, `service.instance`. An entity type is one or more
165/// dot-separated segments, each a non-empty `[a-z0-9_]+` token. The dotted form
166/// carries the two-entity-layer convention (`service` vs `service.instance`).
167fn is_valid_entity_type(ty: &str) -> bool {
168    !ty.is_empty()
169        && ty.split('.').all(|seg| {
170            !seg.is_empty()
171                && seg
172                    .bytes()
173                    .all(|b| b.is_ascii_lowercase() || b.is_ascii_digit() || b == b'_')
174        })
175}
176
177/// Parses a well-formed entity-identity option key of the shape
178/// `greptime.semantic.entity.<type>.{id|descriptive|scope}` into
179/// `(entity_type, role)`. The `<type>` may itself contain dots; the role is the
180/// final dot-separated segment. This is the single parser of the key format —
181/// DDL validation and the read-time derivation both go through it.
182pub fn parse_entity_option_key(key: &str) -> Option<(&str, EntityRole)> {
183    let rest = key.strip_prefix(SEMANTIC_ENTITY_PREFIX)?;
184    let (ty, role) = rest.rsplit_once('.')?;
185    if !is_valid_entity_type(ty) {
186        return None;
187    }
188    Some((ty, EntityRole::parse(role)?))
189}
190
191/// Returns true if `key` is a well-formed entity-identity option key.
192pub fn is_entity_option_key(key: &str) -> bool {
193    parse_entity_option_key(key).is_some()
194}
195
196/// Returns true if a column of `data_type` renders as a stable string — the
197/// requirement for entity id/descriptive/scope columns. The read-time
198/// derivation casts them to strings, so a type without a stable string form
199/// would fail only when the graph is scanned; DDL validation rejects it up
200/// front instead.
201pub fn has_stable_string_form(data_type: &ConcreteDataType) -> bool {
202    !matches!(
203        data_type,
204        ConcreteDataType::Binary(_)
205            | ConcreteDataType::Json(_)
206            | ConcreteDataType::Vector(_)
207            | ConcreteDataType::List(_)
208            | ConcreteDataType::Struct(_)
209            | ConcreteDataType::Dictionary(_)
210            | ConcreteDataType::Null(_)
211    )
212}
213
214/// Tokenizes an entity option's comma-separated column list (trimmed, empty
215/// tokens dropped). [`validate_semantic_option`] rejects empty tokens at DDL
216/// time, so readers only ever drop what validation already refused.
217pub fn parse_entity_columns(value: &str) -> Vec<String> {
218    value
219        .split(',')
220        .map(|c| c.trim().to_string())
221        .filter(|c| !c.is_empty())
222        .collect()
223}
224
225/// Returns true if `key` is a recognised semantic table-option key.
226///
227/// Two acceptance rules: membership in the closed [`SEMANTIC_OPTION_KEYS`]
228/// whitelist, OR the open entity sub-namespace ([`is_entity_option_key`], validated
229/// by prefix + shape). Everything else under [`SEMANTIC_PREFIX`] is rejected, and
230/// the internal [`SEMANTIC_PER_TABLE_INDEX_KEY`] (outside the prefix) never matches.
231pub fn is_semantic_option_key(key: &str) -> bool {
232    SEMANTIC_OPTION_KEYS.contains(&key) || is_entity_option_key(key)
233}
234
235/// Validates a `greptime.semantic.*` option's `value` against its allowed domain.
236///
237/// Open-value keys (unit, original_name, pipeline, conventions) accept any
238/// non-empty string. Closed-domain keys accept a fixed set, plus the `unknown`
239/// sentinel, plus `mixed` for the keys where one long-lived table can
240/// legitimately see multiple values. Entity keys ([`is_entity_option_key`]) take a
241/// comma-separated column-name list (each token non-empty); column existence and
242/// the stable-string-form rule for entity columns are enforced later against
243/// the table schema at DDL time, not here. Keys that are neither whitelisted nor a well-formed entity key
244/// are rejected.
245pub fn validate_semantic_option(key: &str, value: &str) -> bool {
246    if is_entity_option_key(key) {
247        return !value.is_empty() && value.split(',').all(|col| !col.trim().is_empty());
248    }
249    match key {
250        SEMANTIC_PIPELINE
251        | SEMANTIC_SOURCE_VERSION
252        | SEMANTIC_METRIC_UNIT
253        | SEMANTIC_METRIC_ORIGINAL_NAME
254        | SEMANTIC_TRACE_CONVENTIONS => !value.is_empty(),
255
256        SEMANTIC_SIGNAL_TYPE => matches!(value, "trace" | "log" | "metric" | "event" | "unknown"),
257        SEMANTIC_SOURCE => matches!(
258            value,
259            "opentelemetry"
260                | "prometheus"
261                | "influxdb"
262                | "opentsdb"
263                | "elasticsearch"
264                | "loki"
265                | "custom"
266                | "mixed"
267                | "unknown"
268        ),
269        SEMANTIC_METRIC_TYPE => matches!(
270            value,
271            "counter"
272                | "gauge"
273                | "histogram"
274                | "summary"
275                | "updown_counter"
276                | "gauge_histogram"
277                | "info"
278                | "stateset"
279                | "mixed"
280                | "unknown"
281        ),
282        SEMANTIC_METRIC_TEMPORALITY => {
283            matches!(
284                value,
285                METRIC_TEMPORALITY_CUMULATIVE | METRIC_TEMPORALITY_DELTA | "mixed" | "unknown"
286            )
287        }
288        SEMANTIC_METRIC_METADATA_QUALITY => matches!(value, "declared" | "inferred" | "unknown"),
289
290        _ => false,
291    }
292}
293
294#[cfg(test)]
295mod tests {
296    use super::*;
297
298    #[test]
299    fn test_is_semantic_option_key() {
300        assert!(is_semantic_option_key(SEMANTIC_SIGNAL_TYPE));
301        assert!(is_semantic_option_key(SEMANTIC_METRIC_TYPE));
302        assert!(is_semantic_option_key(SEMANTIC_PIPELINE));
303
304        // Unknown keys under the prefix are not whitelisted.
305        assert!(!is_semantic_option_key("greptime.semantic.future.key"));
306        assert!(!is_semantic_option_key("greptime.semantic.unknown_key"));
307        // Keys cut from the vocabulary are no longer accepted.
308        assert!(!is_semantic_option_key(
309            "greptime.semantic.metric.monotonic"
310        ));
311        assert!(!is_semantic_option_key(
312            "greptime.semantic.resource.attributes_dropped"
313        ));
314        // Near-misses must not match.
315        assert!(!is_semantic_option_key("greptime.semanticx"));
316        assert!(!is_semantic_option_key("semantic.signal_type"));
317        assert!(!is_semantic_option_key("table_data_model"));
318        // The internal transport key must never be treated as a table option.
319        assert!(!is_semantic_option_key(SEMANTIC_PER_TABLE_INDEX_KEY));
320    }
321
322    #[test]
323    fn test_validate_semantic_option() {
324        // Enum keys reject out-of-domain values.
325        assert!(validate_semantic_option(SEMANTIC_SIGNAL_TYPE, "metric"));
326        assert!(!validate_semantic_option(SEMANTIC_SIGNAL_TYPE, "spans"));
327        assert!(validate_semantic_option(SEMANTIC_METRIC_TYPE, "counter"));
328        assert!(validate_semantic_option(SEMANTIC_METRIC_TYPE, "mixed"));
329        assert!(!validate_semantic_option(SEMANTIC_METRIC_TYPE, "bogus"));
330
331        // Sentinels and open values.
332        assert!(validate_semantic_option(
333            SEMANTIC_METRIC_TEMPORALITY,
334            "unknown"
335        ));
336        assert!(validate_semantic_option(SEMANTIC_METRIC_UNIT, "By"));
337        assert!(!validate_semantic_option(SEMANTIC_METRIC_UNIT, ""));
338        assert!(validate_semantic_option(
339            SEMANTIC_PIPELINE,
340            "greptime_trace_v1"
341        ));
342
343        // A cut key validates to false regardless of value.
344        assert!(!validate_semantic_option(
345            "greptime.semantic.metric.monotonic",
346            "true"
347        ));
348        // Unknown key is rejected regardless of value.
349        assert!(!validate_semantic_option(
350            "greptime.semantic.future.key",
351            "x"
352        ));
353
354        // Drift guard: every value stamped by the ingestion path must validate.
355        assert!(validate_semantic_option(
356            SEMANTIC_SIGNAL_TYPE,
357            SIGNAL_TYPE_TRACE
358        ));
359        assert!(validate_semantic_option(
360            SEMANTIC_SIGNAL_TYPE,
361            SIGNAL_TYPE_METRIC
362        ));
363        assert!(validate_semantic_option(
364            SEMANTIC_SIGNAL_TYPE,
365            SIGNAL_TYPE_LOG
366        ));
367        assert!(validate_semantic_option(
368            SEMANTIC_SOURCE,
369            SOURCE_OPENTELEMETRY
370        ));
371        assert!(validate_semantic_option(SEMANTIC_SOURCE, SOURCE_PROMETHEUS));
372        assert!(validate_semantic_option(SEMANTIC_SOURCE_VERSION, "2.0"));
373        assert!(validate_semantic_option(
374            SEMANTIC_METRIC_METADATA_QUALITY,
375            METADATA_QUALITY_INFERRED
376        ));
377        assert!(validate_semantic_option(
378            SEMANTIC_METRIC_METADATA_QUALITY,
379            METADATA_QUALITY_DECLARED
380        ));
381        assert!(validate_semantic_option(
382            SEMANTIC_TRACE_CONVENTIONS,
383            SEMANTIC_VALUE_UNKNOWN
384        ));
385        // An empty value never validates, for any whitelisted key.
386        for key in SEMANTIC_OPTION_KEYS {
387            assert!(
388                !validate_semantic_option(key, ""),
389                "empty value should never validate for {key}"
390            );
391        }
392    }
393
394    #[test]
395    fn test_entity_option_key() {
396        // Well-formed entity keys are accepted by the open sub-namespace, and are
397        // therefore recognised as semantic option keys.
398        for key in [
399            "greptime.semantic.entity.service.id",
400            "greptime.semantic.entity.k8s.pod.id",
401            "greptime.semantic.entity.service.instance.descriptive",
402            "greptime.semantic.entity.host.scope",
403            "greptime.semantic.entity.agent.id",
404        ] {
405            assert!(is_entity_option_key(key), "should accept {key}");
406            assert!(is_semantic_option_key(key), "should recognise {key}");
407        }
408
409        // Malformed: empty type segment, bogus role, missing role, bare prefix,
410        // invalid (uppercase) charset in the type.
411        for key in [
412            "greptime.semantic.entity..id",
413            "greptime.semantic.entity.service.bogusrole",
414            "greptime.semantic.entity.service",
415            "greptime.semantic.entity.",
416            "greptime.semantic.entity.Service.id",
417        ] {
418            assert!(!is_entity_option_key(key), "should reject {key}");
419            assert!(!is_semantic_option_key(key), "should not recognise {key}");
420        }
421
422        // A non-entity semantic key is not an entity key.
423        assert!(!is_entity_option_key(SEMANTIC_SIGNAL_TYPE));
424
425        // Drift guard: the auto-stamped constant is a well-formed entity key.
426        assert!(is_entity_option_key(SEMANTIC_ENTITY_SERVICE_ID));
427        assert!(is_semantic_option_key(SEMANTIC_ENTITY_SERVICE_ID));
428    }
429
430    #[test]
431    fn test_validate_entity_option() {
432        // Single- and composite-id column lists validate.
433        assert!(validate_semantic_option(
434            "greptime.semantic.entity.service.id",
435            "service_name"
436        ));
437        assert!(validate_semantic_option(
438            "greptime.semantic.entity.process.id",
439            "pid,start_time"
440        ));
441        // Empty value and blank/empty tokens do not.
442        assert!(!validate_semantic_option(
443            "greptime.semantic.entity.service.id",
444            ""
445        ));
446        assert!(!validate_semantic_option(
447            "greptime.semantic.entity.process.id",
448            "pid,"
449        ));
450        // A malformed entity key validates to false regardless of value.
451        assert!(!validate_semantic_option(
452            "greptime.semantic.entity.service.bogusrole",
453            "service_name"
454        ));
455    }
456}
table/requests/semantic.rs

table/requests/
semantic.rs
