rustc_lexer/
unescape.rs

Help
1//! Utilities for validating string and char literals and turning them into
2//! values they represent.
3
4use std::ops::Range;
5use std::str::Chars;
6
7use Mode::*;
8
9#[cfg(test)]
10mod tests;
11
12/// Errors and warnings that can occur during string unescaping. They mostly
13/// relate to malformed escape sequences, but there are a few that are about
14/// other problems.
15#[derive(Debug, PartialEq, Eq)]
16pub enum EscapeError {
17    /// Expected 1 char, but 0 were found.
18    ZeroChars,
19    /// Expected 1 char, but more than 1 were found.
20    MoreThanOneChar,
21
22    /// Escaped '\' character without continuation.
23    LoneSlash,
24    /// Invalid escape character (e.g. '\z').
25    InvalidEscape,
26    /// Raw '\r' encountered.
27    BareCarriageReturn,
28    /// Raw '\r' encountered in raw string.
29    BareCarriageReturnInRawString,
30    /// Unescaped character that was expected to be escaped (e.g. raw '\t').
31    EscapeOnlyChar,
32
33    /// Numeric character escape is too short (e.g. '\x1').
34    TooShortHexEscape,
35    /// Invalid character in numeric escape (e.g. '\xz')
36    InvalidCharInHexEscape,
37    /// Character code in numeric escape is non-ascii (e.g. '\xFF').
38    OutOfRangeHexEscape,
39
40    /// '\u' not followed by '{'.
41    NoBraceInUnicodeEscape,
42    /// Non-hexadecimal value in '\u{..}'.
43    InvalidCharInUnicodeEscape,
44    /// '\u{}'
45    EmptyUnicodeEscape,
46    /// No closing brace in '\u{..}', e.g. '\u{12'.
47    UnclosedUnicodeEscape,
48    /// '\u{_12}'
49    LeadingUnderscoreUnicodeEscape,
50    /// More than 6 characters in '\u{..}', e.g. '\u{10FFFF_FF}'
51    OverlongUnicodeEscape,
52    /// Invalid in-bound unicode character code, e.g. '\u{DFFF}'.
53    LoneSurrogateUnicodeEscape,
54    /// Out of bounds unicode character code, e.g. '\u{FFFFFF}'.
55    OutOfRangeUnicodeEscape,
56
57    /// Unicode escape code in byte literal.
58    UnicodeEscapeInByte,
59    /// Non-ascii character in byte literal, byte string literal, or raw byte string literal.
60    NonAsciiCharInByte,
61
62    // `\0` in a C string literal.
63    NulInCStr,
64
65    /// After a line ending with '\', the next line contains whitespace
66    /// characters that are not skipped.
67    UnskippedWhitespaceWarning,
68
69    /// After a line ending with '\', multiple lines are skipped.
70    MultipleSkippedLinesWarning,
71}
72
73impl EscapeError {
74    /// Returns true for actual errors, as opposed to warnings.
75    pub fn is_fatal(&self) -> bool {
76        !matches!(
77            self,
78            EscapeError::UnskippedWhitespaceWarning | EscapeError::MultipleSkippedLinesWarning
79        )
80    }
81}
82
83/// Takes the contents of a unicode-only (non-mixed-utf8) literal (without
84/// quotes) and produces a sequence of escaped characters or errors.
85///
86/// Values are returned by invoking `callback`. For `Char` and `Byte` modes,
87/// the callback will be called exactly once.
88pub fn unescape_unicode<F>(src: &str, mode: Mode, callback: &mut F)
89where
90    F: FnMut(Range<usize>, Result<char, EscapeError>),
91{
92    match mode {
93        Char | Byte => {
94            let mut chars = src.chars();
95            let res = unescape_char_or_byte(&mut chars, mode);
96            callback(0..(src.len() - chars.as_str().len()), res);
97        }
98        Str | ByteStr => unescape_non_raw_common(src, mode, callback),
99        RawStr | RawByteStr => check_raw_common(src, mode, callback),
100        RawCStr => check_raw_common(src, mode, &mut |r, mut result| {
101            if let Ok('\0') = result {
102                result = Err(EscapeError::NulInCStr);
103            }
104            callback(r, result)
105        }),
106        CStr => unreachable!(),
107    }
108}
109
110/// Used for mixed utf8 string literals, i.e. those that allow both unicode
111/// chars and high bytes.
112pub enum MixedUnit {
113    /// Used for ASCII chars (written directly or via `\x00`..`\x7f` escapes)
114    /// and Unicode chars (written directly or via `\u` escapes).
115    ///
116    /// For example, if '¥' appears in a string it is represented here as
117    /// `MixedUnit::Char('¥')`, and it will be appended to the relevant byte
118    /// string as the two-byte UTF-8 sequence `[0xc2, 0xa5]`
119    Char(char),
120
121    /// Used for high bytes (`\x80`..`\xff`).
122    ///
123    /// For example, if `\xa5` appears in a string it is represented here as
124    /// `MixedUnit::HighByte(0xa5)`, and it will be appended to the relevant
125    /// byte string as the single byte `0xa5`.
126    HighByte(u8),
127}
128
129impl From<char> for MixedUnit {
130    fn from(c: char) -> Self {
131        MixedUnit::Char(c)
132    }
133}
134
135impl From<u8> for MixedUnit {
136    fn from(n: u8) -> Self {
137        if n.is_ascii() { MixedUnit::Char(n as char) } else { MixedUnit::HighByte(n) }
138    }
139}
140
141/// Takes the contents of a mixed-utf8 literal (without quotes) and produces
142/// a sequence of escaped characters or errors.
143///
144/// Values are returned by invoking `callback`.
145pub fn unescape_mixed<F>(src: &str, mode: Mode, callback: &mut F)
146where
147    F: FnMut(Range<usize>, Result<MixedUnit, EscapeError>),
148{
149    match mode {
150        CStr => unescape_non_raw_common(src, mode, &mut |r, mut result| {
151            if let Ok(MixedUnit::Char('\0')) = result {
152                result = Err(EscapeError::NulInCStr);
153            }
154            callback(r, result)
155        }),
156        Char | Byte | Str | RawStr | ByteStr | RawByteStr | RawCStr => unreachable!(),
157    }
158}
159
160/// Takes a contents of a char literal (without quotes), and returns an
161/// unescaped char or an error.
162pub fn unescape_char(src: &str) -> Result<char, EscapeError> {
163    unescape_char_or_byte(&mut src.chars(), Char)
164}
165
166/// Takes a contents of a byte literal (without quotes), and returns an
167/// unescaped byte or an error.
168pub fn unescape_byte(src: &str) -> Result<u8, EscapeError> {
169    unescape_char_or_byte(&mut src.chars(), Byte).map(byte_from_char)
170}
171
172/// What kind of literal do we parse.
173#[derive(Debug, Clone, Copy, PartialEq)]
174pub enum Mode {
175    Char,
176
177    Byte,
178
179    Str,
180    RawStr,
181
182    ByteStr,
183    RawByteStr,
184
185    CStr,
186    RawCStr,
187}
188
189impl Mode {
190    pub fn in_double_quotes(self) -> bool {
191        match self {
192            Str | RawStr | ByteStr | RawByteStr | CStr | RawCStr => true,
193            Char | Byte => false,
194        }
195    }
196
197    /// Are `\x80`..`\xff` allowed?
198    fn allow_high_bytes(self) -> bool {
199        match self {
200            Char | Str => false,
201            Byte | ByteStr | CStr => true,
202            RawStr | RawByteStr | RawCStr => unreachable!(),
203        }
204    }
205
206    /// Are unicode (non-ASCII) chars allowed?
207    #[inline]
208    fn allow_unicode_chars(self) -> bool {
209        match self {
210            Byte | ByteStr | RawByteStr => false,
211            Char | Str | RawStr | CStr | RawCStr => true,
212        }
213    }
214
215    /// Are unicode escapes (`\u`) allowed?
216    fn allow_unicode_escapes(self) -> bool {
217        match self {
218            Byte | ByteStr => false,
219            Char | Str | CStr => true,
220            RawByteStr | RawStr | RawCStr => unreachable!(),
221        }
222    }
223
224    pub fn prefix_noraw(self) -> &'static str {
225        match self {
226            Char | Str | RawStr => "",
227            Byte | ByteStr | RawByteStr => "b",
228            CStr | RawCStr => "c",
229        }
230    }
231}
232
233fn scan_escape<T: From<char> + From<u8>>(
234    chars: &mut Chars<'_>,
235    mode: Mode,
236) -> Result<T, EscapeError> {
237    // Previous character was '\\', unescape what follows.
238    let res: char = match chars.next().ok_or(EscapeError::LoneSlash)? {
239        '"' => '"',
240        'n' => '\n',
241        'r' => '\r',
242        't' => '\t',
243        '\\' => '\\',
244        '\'' => '\'',
245        '0' => '\0',
246        'x' => {
247            // Parse hexadecimal character code.
248
249            let hi = chars.next().ok_or(EscapeError::TooShortHexEscape)?;
250            let hi = hi.to_digit(16).ok_or(EscapeError::InvalidCharInHexEscape)?;
251
252            let lo = chars.next().ok_or(EscapeError::TooShortHexEscape)?;
253            let lo = lo.to_digit(16).ok_or(EscapeError::InvalidCharInHexEscape)?;
254
255            let value = (hi * 16 + lo) as u8;
256
257            return if !mode.allow_high_bytes() && !value.is_ascii() {
258                Err(EscapeError::OutOfRangeHexEscape)
259            } else {
260                // This may be a high byte, but that will only happen if `T` is
261                // `MixedUnit`, because of the `allow_high_bytes` check above.
262                Ok(T::from(value))
263            };
264        }
265        'u' => return scan_unicode(chars, mode.allow_unicode_escapes()).map(T::from),
266        _ => return Err(EscapeError::InvalidEscape),
267    };
268    Ok(T::from(res))
269}
270
271fn scan_unicode(chars: &mut Chars<'_>, allow_unicode_escapes: bool) -> Result<char, EscapeError> {
272    // We've parsed '\u', now we have to parse '{..}'.
273
274    if chars.next() != Some('{') {
275        return Err(EscapeError::NoBraceInUnicodeEscape);
276    }
277
278    // First character must be a hexadecimal digit.
279    let mut n_digits = 1;
280    let mut value: u32 = match chars.next().ok_or(EscapeError::UnclosedUnicodeEscape)? {
281        '_' => return Err(EscapeError::LeadingUnderscoreUnicodeEscape),
282        '}' => return Err(EscapeError::EmptyUnicodeEscape),
283        c => c.to_digit(16).ok_or(EscapeError::InvalidCharInUnicodeEscape)?,
284    };
285
286    // First character is valid, now parse the rest of the number
287    // and closing brace.
288    loop {
289        match chars.next() {
290            None => return Err(EscapeError::UnclosedUnicodeEscape),
291            Some('_') => continue,
292            Some('}') => {
293                if n_digits > 6 {
294                    return Err(EscapeError::OverlongUnicodeEscape);
295                }
296
297                // Incorrect syntax has higher priority for error reporting
298                // than unallowed value for a literal.
299                if !allow_unicode_escapes {
300                    return Err(EscapeError::UnicodeEscapeInByte);
301                }
302
303                break std::char::from_u32(value).ok_or({
304                    if value > 0x10FFFF {
305                        EscapeError::OutOfRangeUnicodeEscape
306                    } else {
307                        EscapeError::LoneSurrogateUnicodeEscape
308                    }
309                });
310            }
311            Some(c) => {
312                let digit: u32 = c.to_digit(16).ok_or(EscapeError::InvalidCharInUnicodeEscape)?;
313                n_digits += 1;
314                if n_digits > 6 {
315                    // Stop updating value since we're sure that it's incorrect already.
316                    continue;
317                }
318                value = value * 16 + digit;
319            }
320        };
321    }
322}
323
324#[inline]
325fn ascii_check(c: char, allow_unicode_chars: bool) -> Result<char, EscapeError> {
326    if allow_unicode_chars || c.is_ascii() { Ok(c) } else { Err(EscapeError::NonAsciiCharInByte) }
327}
328
329fn unescape_char_or_byte(chars: &mut Chars<'_>, mode: Mode) -> Result<char, EscapeError> {
330    let c = chars.next().ok_or(EscapeError::ZeroChars)?;
331    let res = match c {
332        '\\' => scan_escape(chars, mode),
333        '\n' | '\t' | '\'' => Err(EscapeError::EscapeOnlyChar),
334        '\r' => Err(EscapeError::BareCarriageReturn),
335        _ => ascii_check(c, mode.allow_unicode_chars()),
336    }?;
337    if chars.next().is_some() {
338        return Err(EscapeError::MoreThanOneChar);
339    }
340    Ok(res)
341}
342
343/// Takes a contents of a string literal (without quotes) and produces a
344/// sequence of escaped characters or errors.
345fn unescape_non_raw_common<F, T: From<char> + From<u8>>(src: &str, mode: Mode, callback: &mut F)
346where
347    F: FnMut(Range<usize>, Result<T, EscapeError>),
348{
349    let mut chars = src.chars();
350    let allow_unicode_chars = mode.allow_unicode_chars(); // get this outside the loop
351
352    // The `start` and `end` computation here is complicated because
353    // `skip_ascii_whitespace` makes us to skip over chars without counting
354    // them in the range computation.
355    while let Some(c) = chars.next() {
356        let start = src.len() - chars.as_str().len() - c.len_utf8();
357        let res = match c {
358            '\\' => {
359                match chars.clone().next() {
360                    Some('\n') => {
361                        // Rust language specification requires us to skip whitespaces
362                        // if unescaped '\' character is followed by '\n'.
363                        // For details see [Rust language reference]
364                        // (https://doc.rust-lang.org/reference/tokens.html#string-literals).
365                        skip_ascii_whitespace(&mut chars, start, &mut |range, err| {
366                            callback(range, Err(err))
367                        });
368                        continue;
369                    }
370                    _ => scan_escape::<T>(&mut chars, mode),
371                }
372            }
373            '"' => Err(EscapeError::EscapeOnlyChar),
374            '\r' => Err(EscapeError::BareCarriageReturn),
375            _ => ascii_check(c, allow_unicode_chars).map(T::from),
376        };
377        let end = src.len() - chars.as_str().len();
378        callback(start..end, res);
379    }
380}
381
382fn skip_ascii_whitespace<F>(chars: &mut Chars<'_>, start: usize, callback: &mut F)
383where
384    F: FnMut(Range<usize>, EscapeError),
385{
386    let tail = chars.as_str();
387    let first_non_space = tail
388        .bytes()
389        .position(|b| b != b' ' && b != b'\t' && b != b'\n' && b != b'\r')
390        .unwrap_or(tail.len());
391    if tail[1..first_non_space].contains('\n') {
392        // The +1 accounts for the escaping slash.
393        let end = start + first_non_space + 1;
394        callback(start..end, EscapeError::MultipleSkippedLinesWarning);
395    }
396    let tail = &tail[first_non_space..];
397    if let Some(c) = tail.chars().next() {
398        if c.is_whitespace() {
399            // For error reporting, we would like the span to contain the character that was not
400            // skipped. The +1 is necessary to account for the leading \ that started the escape.
401            let end = start + first_non_space + c.len_utf8() + 1;
402            callback(start..end, EscapeError::UnskippedWhitespaceWarning);
403        }
404    }
405    *chars = tail.chars();
406}
407
408/// Takes a contents of a string literal (without quotes) and produces a
409/// sequence of characters or errors.
410/// NOTE: Raw strings do not perform any explicit character escaping, here we
411/// only produce errors on bare CR.
412fn check_raw_common<F>(src: &str, mode: Mode, callback: &mut F)
413where
414    F: FnMut(Range<usize>, Result<char, EscapeError>),
415{
416    let mut chars = src.chars();
417    let allow_unicode_chars = mode.allow_unicode_chars(); // get this outside the loop
418
419    // The `start` and `end` computation here matches the one in
420    // `unescape_non_raw_common` for consistency, even though this function
421    // doesn't have to worry about skipping any chars.
422    while let Some(c) = chars.next() {
423        let start = src.len() - chars.as_str().len() - c.len_utf8();
424        let res = match c {
425            '\r' => Err(EscapeError::BareCarriageReturnInRawString),
426            _ => ascii_check(c, allow_unicode_chars),
427        };
428        let end = src.len() - chars.as_str().len();
429        callback(start..end, res);
430    }
431}
432
433#[inline]
434pub fn byte_from_char(c: char) -> u8 {
435    let res = c as u32;
436    debug_assert!(res <= u8::MAX as u32, "guaranteed because of ByteStr");
437    res as u8
438}
rustc_lexer/unescape.rs

rustc_lexer/
unescape.rs
