common_meta/kv_backend/
util.rs

1// Copyright 2023 Greptime Team
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14
15use url::Url;
16
17/// Placeholder substituted for any password found in a connection string.
18const REDACTED: &str = "***";
19
20/// Substituted for a whole connection string that cannot be safely sanitized.
21const REDACTED_CONNECTION_STRING: &str = "<redacted connection string>";
22
23/// Removes sensitive information (passwords) from a connection string before it
24/// is logged.
25///
26/// `store_addrs` entries can be a PostgreSQL DSN (a `postgres(ql)://` URL or a
27/// libpq keyword string), a MySQL/etcd URL, or an etcd host list.
28///
29/// The function fails closed: it never returns a URI-shaped input verbatim. In
30/// order:
31/// 1. PostgreSQL DSNs are parsed with `tokio_postgres::Config` — the backend's
32///    own parser — and logged via its `Debug`, which redacts the password. This
33///    matches the grammar exactly (multi-host URIs, `\`-escapes, any Unicode
34///    whitespace, percent-encoded query keys, `&`/`;`/`://` inside values).
35/// 2. Any other URL is redacted with the `url` crate.
36/// 3. A URI-shaped input that neither parser accepted (e.g. a malformed
37///    multi-host Postgres URI, with or without leading whitespace) is redacted
38///    in full — credentials in an unparsable authority or query cannot be split
39///    out reliably, so the whole string is replaced rather than reparsed.
40/// 4. Anything else is treated as a keyword string and best-effort redacted.
41pub fn sanitize_connection_string(conn_str: &str) -> String {
42    // Redact explicit keyword-style password assignments before parsing. A
43    // malformed string such as `host=x;password=secret` can otherwise parse as
44    // a host value, causing `Config`'s Debug output to retain the password text.
45    if !is_uri_like(conn_str) {
46        let redacted = redact_keyword_password(conn_str);
47        if redacted != conn_str {
48            return redacted;
49        }
50    }
51
52    #[cfg(feature = "pg_kvbackend")]
53    if let Ok(config) = conn_str.parse::<tokio_postgres::Config>() {
54        // `Config`'s Debug prints the password as `Some("REDACTED")`.
55        return format!("{config:?}");
56    }
57
58    if let Ok(url) = Url::parse(conn_str) {
59        return redact_url(url);
60    }
61
62    if is_uri_like(conn_str) {
63        return REDACTED_CONNECTION_STRING.to_string();
64    }
65
66    redact_keyword_password(conn_str)
67}
68
69/// Redacts the userinfo password and any `password` query parameter of a parsed URL.
70fn redact_url(mut url: Url) -> String {
71    if url.password().is_some() {
72        let _ = url.set_password(Some(REDACTED));
73    }
74    if url.query_pairs().any(|(key, _)| key == "password") {
75        let redacted: Vec<(String, String)> = url
76            .query_pairs()
77            .map(|(key, value)| {
78                let value = if key == "password" {
79                    REDACTED.to_string()
80                } else {
81                    value.into_owned()
82                };
83                (key.into_owned(), value)
84            })
85            .collect();
86        url.query_pairs_mut().clear().extend_pairs(redacted);
87    }
88    url.to_string()
89}
90
91/// True if `s`, ignoring leading whitespace, begins with a valid URI scheme
92/// followed by `://` (`scheme = ALPHA *( ALPHA / DIGIT / "+" / "-" / "." )`).
93/// Leading whitespace (including Unicode, e.g. U+2003) does not stop a value from
94/// being a URI, so it is trimmed first; a bare `contains("://")` would instead
95/// misfire on a `://` inside a keyword value.
96fn is_uri_like(s: &str) -> bool {
97    let s = s.trim_start();
98    let Some(pos) = s.find("://") else {
99        return false;
100    };
101    let mut chars = s[..pos].chars();
102    matches!(chars.next(), Some(c) if c.is_ascii_alphabetic())
103        && chars.all(|c| c.is_ascii_alphanumeric() || matches!(c, '+' | '-' | '.'))
104}
105
106/// Redacts `password` values in a keyword string. A standalone, case-insensitive
107/// `password` assignment outside a recognized keyword boundary causes the whole
108/// string to be redacted, since another option's value may otherwise retain it.
109/// Keys are separated by Unicode whitespace; a value is single/double quoted or
110/// runs to the next unescaped whitespace, with `\` escaping the next character.
111fn redact_keyword_password(s: &str) -> String {
112    const KEY: &str = "password";
113    let mut out = String::with_capacity(s.len());
114    let mut i = 0;
115    while i < s.len() {
116        if keyword_matches_at(s, i, KEY) {
117            let mut j = skip_ws(s, i + KEY.len());
118            if s[j..].starts_with('=') {
119                if is_keyword_at(s, i, KEY) {
120                    j = skip_ws(s, j + 1);
121                    out.push_str("password=");
122                    out.push_str(REDACTED);
123                    i = skip_value(s, j);
124                    continue;
125                }
126
127                let embedded_in_identifier = s[..i]
128                    .chars()
129                    .next_back()
130                    .is_some_and(|c| c.is_alphanumeric() || c == '_');
131                if !embedded_in_identifier {
132                    return REDACTED_CONNECTION_STRING.to_string();
133                }
134            }
135        }
136        let ch = s[i..].chars().next().unwrap();
137        out.push(ch);
138        i += ch.len_utf8();
139    }
140    out
141}
142
143fn keyword_matches_at(s: &str, i: usize, keyword: &str) -> bool {
144    s[i..]
145        .get(..keyword.len())
146        .is_some_and(|candidate| candidate.eq_ignore_ascii_case(keyword))
147}
148
149fn skip_ws(s: &str, mut i: usize) -> usize {
150    while let Some(ch) = s[i..].chars().next() {
151        if ch.is_whitespace() {
152            i += ch.len_utf8();
153        } else {
154            break;
155        }
156    }
157    i
158}
159
160/// True if `keyword` starts at byte `i` preceded by a keyword boundary. In
161/// addition to valid libpq whitespace separators, `;` and `&` are treated as
162/// conservative boundaries so malformed DSNs cannot leak credentials. Other
163/// punctuation, such as the `=` in `application_name=password=x`, is handled by
164/// the fail-closed path.
165fn is_keyword_at(s: &str, i: usize, keyword: &str) -> bool {
166    keyword_matches_at(s, i, keyword)
167        && (i == 0
168            || s[..i]
169                .chars()
170                .next_back()
171                .is_some_and(|c| c.is_whitespace() || matches!(c, ';' | '&')))
172}
173
174/// Returns the byte index just past a keyword value beginning at `start`.
175fn skip_value(s: &str, start: usize) -> usize {
176    if start >= s.len() {
177        return start;
178    }
179    let quote = s[start..].chars().next().unwrap();
180    let quoted = quote == '\'' || quote == '"';
181    let mut i = if quoted {
182        start + quote.len_utf8()
183    } else {
184        start
185    };
186    while i < s.len() {
187        let ch = s[i..].chars().next().unwrap();
188        if ch == '\\' {
189            i += ch.len_utf8();
190            if let Some(next) = s[i..].chars().next() {
191                i += next.len_utf8(); // skip the escaped character
192            }
193            continue;
194        }
195        if quoted {
196            if ch == quote {
197                return i + ch.len_utf8(); // include the closing quote
198            }
199        } else if ch.is_whitespace() {
200            return i;
201        }
202        i += ch.len_utf8();
203    }
204    s.len()
205}
206
207#[cfg(test)]
208mod tests {
209    use super::*;
210
211    #[test]
212    fn test_sanitize_non_pg_url() {
213        let s = sanitize_connection_string("mysql://user:password123@localhost:3306/db");
214        assert!(!s.contains("password123"), "{s}");
215        assert!(s.contains("user:***@localhost:3306"), "{s}");
216
217        let s = sanitize_connection_string("mysql://localhost:3306/db");
218        assert!(s.contains("localhost:3306"), "{s}");
219
220        let s = sanitize_connection_string("mysql://user@localhost:3306/db?password=secret");
221        assert!(!s.contains("secret"), "{s}");
222
223        let s = sanitize_connection_string("http://etcd-host:2379");
224        assert!(s.contains("etcd-host:2379"), "{s}");
225    }
226
227    // Fail-closed: a URI-shaped string that neither parser accepts is redacted in
228    // full. These multi-host URIs are rejected by both `url` (InvalidPort) and,
229    // when compiled, `tokio_postgres::Config` (unknown option), so the invariant
230    // holds regardless of the `pg_kvbackend` feature — including with leading
231    // (Unicode) whitespace and percent-encoded query keys.
232    #[test]
233    fn test_sanitize_fail_closed_uri() {
234        for dsn in [
235            "postgresql://user:secret@host1:1234,host2:5678/db?unknown=x",
236            " postgresql://user:secret@host1:1234,host2:5678/db?unknown=x",
237            "\u{2003}postgresql://user:secret@host1:1234,host2:5678/db?unknown=x",
238            "postgresql://user@host1:1,host2:2/db?pass%77ord=qsecret&unknown=x",
239        ] {
240            let s = sanitize_connection_string(dsn);
241            // "secret" also matches "qsecret", so this covers the query case too.
242            assert!(!s.contains("secret"), "leaked for {dsn:?}: {s}");
243            // Redacted whole, not reparsed: the authority is gone as well.
244            assert!(!s.contains("host1"), "not fully redacted for {dsn:?}: {s}");
245        }
246    }
247
248    #[test]
249    fn test_sanitize_malformed_keyword_fail_closed() {
250        for dsn in [
251            "host=localhost;password=LEAK_CANARY",
252            "host=localhost&password=LEAK_CANARY",
253        ] {
254            let s = sanitize_connection_string(dsn);
255            assert!(!s.contains("LEAK_CANARY"), "leaked for {dsn:?}: {s}");
256        }
257
258        for dsn in [
259            "host=localhost,password=LEAK_CANARY",
260            "host=localhost/password=LEAK_CANARY",
261            "host=localhost?password=LEAK_CANARY",
262            "host=localhost#password=LEAK_CANARY",
263            "host=localhost:password=LEAK_CANARY",
264            "host=localhost,PASSWORD=LEAK_CANARY",
265            "application_name=password=LEAK_CANARY",
266            "password=a host=localhost,password=LEAK_CANARY",
267            "host=h,password=LEAK_CANARY password=b",
268            "user=u password=a host=h/password=LEAK_CANARY",
269        ] {
270            let s = sanitize_connection_string(dsn);
271            assert_eq!(REDACTED_CONNECTION_STRING, s, "for {dsn:?}");
272        }
273
274        for (dsn, expected) in [
275            (
276                "PASSWORD=LEAK_CANARY host=localhost",
277                "password=*** host=localhost",
278            ),
279            (
280                "host=localhost Password=LEAK_CANARY",
281                "host=localhost password=***",
282            ),
283        ] {
284            let s = sanitize_connection_string(dsn);
285            assert_eq!(expected, s, "for {dsn:?}");
286        }
287
288        for dsn in ["notpassword=LEAK_CANARY", "notPassword=LEAK_CANARY"] {
289            let s = sanitize_connection_string(dsn);
290            assert!(s.contains("LEAK_CANARY"), "over-redacted for {dsn:?}: {s}");
291        }
292    }
293
294    // Keyword redaction must handle libpq quoting and Unicode whitespace without
295    // misfiring on `=`/`://` inside values. Exact output is asserted without the
296    // pg feature; feature-enabled coverage below asserts the no-leak invariant.
297    #[cfg(not(feature = "pg_kvbackend"))]
298    #[test]
299    fn test_sanitize_keyword() {
300        for (dsn, ok) in [
301            (
302                "host=localhost password=secret dbname=x",
303                "host=localhost password=*** dbname=x",
304            ),
305            (
306                "host=localhost password = secret dbname=x",
307                "host=localhost password=*** dbname=x",
308            ),
309            (
310                "host=localhost password='my secret' dbname=x",
311                "host=localhost password=*** dbname=x",
312            ),
313            (
314                "host=localhost password=\"my secret\" dbname=x",
315                "host=localhost password=*** dbname=x",
316            ),
317            (
318                r"host=localhost password='pa\'ss' dbname=x",
319                "host=localhost password=*** dbname=x",
320            ),
321            (
322                "host=localhost user=password dbname=x",
323                "host=localhost user=password dbname=x",
324            ),
325            // A value that merely contains `://` is not treated as a URI.
326            (
327                "host=localhost password=secret application_name=svc://a",
328                "host=localhost password=*** application_name=svc://a",
329            ),
330            // `&` and a `\`-escaped space are part of an unquoted libpq value, so
331            // the whole value is consumed (no trailing suffix leaks).
332            (
333                "host=localhost password=secret&suffix dbname=x",
334                "host=localhost password=*** dbname=x",
335            ),
336            (
337                r"host=localhost password=secret\ suffix dbname=x",
338                "host=localhost password=*** dbname=x",
339            ),
340            // `;` is not a libpq delimiter, so it belongs to the value too.
341            ("password=secret;host=localhost", "password=***"),
342        ] {
343            assert_eq!(sanitize_connection_string(dsn), ok, "for {dsn:?}");
344        }
345
346        // Unicode whitespace as separator / around '=' must not leak.
347        for dsn in [
348            "host=localhost\u{2003}password=secret",
349            "host=localhost password\u{2003}=\u{2003}secret",
350        ] {
351            assert!(
352                !sanitize_connection_string(dsn).contains("secret"),
353                "leaked for {dsn:?}"
354            );
355        }
356    }
357
358    // With the pg backend enabled, cover both the conservative keyword pre-pass
359    // and URI forms parsed by tokio-postgres.
360    #[cfg(feature = "pg_kvbackend")]
361    #[test]
362    fn test_sanitize_pg_dsn() {
363        for dsn in [
364            "postgresql://user:secret@localhost:5432/db",
365            "postgresql://user:secret@host1:1234,host2,host3:5678/db",
366            "postgresql://user@host1:1,host2/db?pass%77ord=secret",
367            "host=localhost port=5432 user=postgres password=secret dbname=mydb",
368            "host=localhost\u{2003}password=secret",
369            "password=secret;host=localhost",
370            "host=localhost password=secret application_name=http://client",
371        ] {
372            assert!(
373                !sanitize_connection_string(dsn).contains("secret"),
374                "leaked for {dsn:?}"
375            );
376        }
377    }
378}
common_meta/kv_backend/util.rs

common_meta/kv_backend/
util.rs
