rustc_lexer/
lib.rs

1//! Low-level Rust lexer.
2//!
3//! The idea with `rustc_lexer` is to make a reusable library,
4//! by separating out pure lexing and rustc-specific concerns, like spans,
5//! error reporting, and interning. So, rustc_lexer operates directly on `&str`,
6//! produces simple tokens which are a pair of type-tag and a bit of original text,
7//! and does not report errors, instead storing them as flags on the token.
8//!
9//! Tokens produced by this lexer are not yet ready for parsing the Rust syntax.
10//! For that see [`rustc_parse::lexer`], which converts this basic token stream
11//! into wide tokens used by actual parser.
12//!
13//! The purpose of this crate is to convert raw sources into a labeled sequence
14//! of well-known token types, so building an actual Rust token stream will
15//! be easier.
16//!
17//! The main entity of this crate is the [`TokenKind`] enum which represents common
18//! lexeme types.
19//!
20//! [`rustc_parse::lexer`]: ../rustc_parse/lexer/index.html
21
22// tidy-alphabetical-start
23// We want to be able to build this crate with a stable compiler,
24// so no `#![feature]` attributes should be added.
25#![deny(unstable_features)]
26// tidy-alphabetical-end
27
28mod cursor;
29
30#[cfg(test)]
31mod tests;
32
33use unicode_properties::UnicodeEmoji;
34pub use unicode_xid::UNICODE_VERSION as UNICODE_XID_VERSION;
35
36use self::LiteralKind::*;
37use self::TokenKind::*;
38pub use crate::cursor::Cursor;
39use crate::cursor::EOF_CHAR;
40
41/// Parsed token.
42/// It doesn't contain information about data that has been parsed,
43/// only the type of the token and its size.
44#[derive(Debug)]
45pub struct Token {
46    pub kind: TokenKind,
47    pub len: u32,
48}
49
50impl Token {
51    fn new(kind: TokenKind, len: u32) -> Token {
52        Token { kind, len }
53    }
54}
55
56/// Enum representing common lexeme types.
57#[derive(Clone, Copy, Debug, PartialEq, Eq)]
58pub enum TokenKind {
59    /// A line comment, e.g. `// comment`.
60    LineComment { doc_style: Option<DocStyle> },
61
62    /// A block comment, e.g. `/* block comment */`.
63    ///
64    /// Block comments can be recursive, so a sequence like `/* /* */`
65    /// will not be considered terminated and will result in a parsing error.
66    BlockComment { doc_style: Option<DocStyle>, terminated: bool },
67
68    /// Any whitespace character sequence.
69    Whitespace,
70
71    /// An identifier or keyword, e.g. `ident` or `continue`.
72    Ident,
73
74    /// An identifier that is invalid because it contains emoji.
75    InvalidIdent,
76
77    /// A raw identifier, e.g. "r#ident".
78    RawIdent,
79
80    /// An unknown literal prefix, like `foo#`, `foo'`, `foo"`. Excludes
81    /// literal prefixes that contain emoji, which are considered "invalid".
82    ///
83    /// Note that only the
84    /// prefix (`foo`) is included in the token, not the separator (which is
85    /// lexed as its own distinct token). In Rust 2021 and later, reserved
86    /// prefixes are reported as errors; in earlier editions, they result in a
87    /// (allowed by default) lint, and are treated as regular identifier
88    /// tokens.
89    UnknownPrefix,
90
91    /// An unknown prefix in a lifetime, like `'foo#`.
92    ///
93    /// Like `UnknownPrefix`, only the `'` and prefix are included in the token
94    /// and not the separator.
95    UnknownPrefixLifetime,
96
97    /// A raw lifetime, e.g. `'r#foo`. In edition < 2021 it will be split into
98    /// several tokens: `'r` and `#` and `foo`.
99    RawLifetime,
100
101    /// Guarded string literal prefix: `#"` or `##`.
102    ///
103    /// Used for reserving "guarded strings" (RFC 3598) in edition 2024.
104    /// Split into the component tokens on older editions.
105    GuardedStrPrefix,
106
107    /// Literals, e.g. `12u8`, `1.0e-40`, `b"123"`. Note that `_` is an invalid
108    /// suffix, but may be present here on string and float literals. Users of
109    /// this type will need to check for and reject that case.
110    ///
111    /// See [LiteralKind] for more details.
112    Literal { kind: LiteralKind, suffix_start: u32 },
113
114    /// A lifetime, e.g. `'a`.
115    Lifetime { starts_with_number: bool },
116
117    /// `;`
118    Semi,
119    /// `,`
120    Comma,
121    /// `.`
122    Dot,
123    /// `(`
124    OpenParen,
125    /// `)`
126    CloseParen,
127    /// `{`
128    OpenBrace,
129    /// `}`
130    CloseBrace,
131    /// `[`
132    OpenBracket,
133    /// `]`
134    CloseBracket,
135    /// `@`
136    At,
137    /// `#`
138    Pound,
139    /// `~`
140    Tilde,
141    /// `?`
142    Question,
143    /// `:`
144    Colon,
145    /// `$`
146    Dollar,
147    /// `=`
148    Eq,
149    /// `!`
150    Bang,
151    /// `<`
152    Lt,
153    /// `>`
154    Gt,
155    /// `-`
156    Minus,
157    /// `&`
158    And,
159    /// `|`
160    Or,
161    /// `+`
162    Plus,
163    /// `*`
164    Star,
165    /// `/`
166    Slash,
167    /// `^`
168    Caret,
169    /// `%`
170    Percent,
171
172    /// Unknown token, not expected by the lexer, e.g. "№"
173    Unknown,
174
175    /// End of input.
176    Eof,
177}
178
179#[derive(Clone, Copy, Debug, PartialEq, Eq)]
180pub enum DocStyle {
181    Outer,
182    Inner,
183}
184
185/// Enum representing the literal types supported by the lexer.
186///
187/// Note that the suffix is *not* considered when deciding the `LiteralKind` in
188/// this type. This means that float literals like `1f32` are classified by this
189/// type as `Int`. (Compare against `rustc_ast::token::LitKind` and
190/// `rustc_ast::ast::LitKind`).
191#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
192pub enum LiteralKind {
193    /// `12_u8`, `0o100`, `0b120i99`, `1f32`.
194    Int { base: Base, empty_int: bool },
195    /// `12.34f32`, `1e3`, but not `1f32`.
196    Float { base: Base, empty_exponent: bool },
197    /// `'a'`, `'\\'`, `'''`, `';`
198    Char { terminated: bool },
199    /// `b'a'`, `b'\\'`, `b'''`, `b';`
200    Byte { terminated: bool },
201    /// `"abc"`, `"abc`
202    Str { terminated: bool },
203    /// `b"abc"`, `b"abc`
204    ByteStr { terminated: bool },
205    /// `c"abc"`, `c"abc`
206    CStr { terminated: bool },
207    /// `r"abc"`, `r#"abc"#`, `r####"ab"###"c"####`, `r#"a`. `None` indicates
208    /// an invalid literal.
209    RawStr { n_hashes: Option<u8> },
210    /// `br"abc"`, `br#"abc"#`, `br####"ab"###"c"####`, `br#"a`. `None`
211    /// indicates an invalid literal.
212    RawByteStr { n_hashes: Option<u8> },
213    /// `cr"abc"`, "cr#"abc"#", `cr#"a`. `None` indicates an invalid literal.
214    RawCStr { n_hashes: Option<u8> },
215}
216
217/// `#"abc"#`, `##"a"` (fewer closing), or even `#"a` (unterminated).
218///
219/// Can capture fewer closing hashes than starting hashes,
220/// for more efficient lexing and better backwards diagnostics.
221#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
222pub struct GuardedStr {
223    pub n_hashes: u32,
224    pub terminated: bool,
225    pub token_len: u32,
226}
227
228#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
229pub enum RawStrError {
230    /// Non `#` characters exist between `r` and `"`, e.g. `r##~"abcde"##`
231    InvalidStarter { bad_char: char },
232    /// The string was not terminated, e.g. `r###"abcde"##`.
233    /// `possible_terminator_offset` is the number of characters after `r` or
234    /// `br` where they may have intended to terminate it.
235    NoTerminator { expected: u32, found: u32, possible_terminator_offset: Option<u32> },
236    /// More than 255 `#`s exist.
237    TooManyDelimiters { found: u32 },
238}
239
240/// Base of numeric literal encoding according to its prefix.
241#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
242pub enum Base {
243    /// Literal starts with "0b".
244    Binary = 2,
245    /// Literal starts with "0o".
246    Octal = 8,
247    /// Literal doesn't contain a prefix.
248    Decimal = 10,
249    /// Literal starts with "0x".
250    Hexadecimal = 16,
251}
252
253/// `rustc` allows files to have a shebang, e.g. "#!/usr/bin/rustrun",
254/// but shebang isn't a part of rust syntax.
255pub fn strip_shebang(input: &str) -> Option<usize> {
256    // Shebang must start with `#!` literally, without any preceding whitespace.
257    // For simplicity we consider any line starting with `#!` a shebang,
258    // regardless of restrictions put on shebangs by specific platforms.
259    if let Some(input_tail) = input.strip_prefix("#!") {
260        // Ok, this is a shebang but if the next non-whitespace token is `[`,
261        // then it may be valid Rust code, so consider it Rust code.
262        let next_non_whitespace_token = tokenize(input_tail).map(|tok| tok.kind).find(|tok| {
263            !matches!(
264                tok,
265                TokenKind::Whitespace
266                    | TokenKind::LineComment { doc_style: None }
267                    | TokenKind::BlockComment { doc_style: None, .. }
268            )
269        });
270        if next_non_whitespace_token != Some(TokenKind::OpenBracket) {
271            // No other choice than to consider this a shebang.
272            return Some(2 + input_tail.lines().next().unwrap_or_default().len());
273        }
274    }
275    None
276}
277
278/// Validates a raw string literal. Used for getting more information about a
279/// problem with a `RawStr`/`RawByteStr` with a `None` field.
280#[inline]
281pub fn validate_raw_str(input: &str, prefix_len: u32) -> Result<(), RawStrError> {
282    debug_assert!(!input.is_empty());
283    let mut cursor = Cursor::new(input);
284    // Move past the leading `r` or `br`.
285    for _ in 0..prefix_len {
286        cursor.bump().unwrap();
287    }
288    cursor.raw_double_quoted_string(prefix_len).map(|_| ())
289}
290
291/// Creates an iterator that produces tokens from the input string.
292pub fn tokenize(input: &str) -> impl Iterator<Item = Token> {
293    let mut cursor = Cursor::new(input);
294    std::iter::from_fn(move || {
295        let token = cursor.advance_token();
296        if token.kind != TokenKind::Eof { Some(token) } else { None }
297    })
298}
299
300/// True if `c` is considered a whitespace according to Rust language definition.
301/// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
302/// for definitions of these classes.
303pub fn is_whitespace(c: char) -> bool {
304    // This is Pattern_White_Space.
305    //
306    // Note that this set is stable (ie, it doesn't change with different
307    // Unicode versions), so it's ok to just hard-code the values.
308
309    matches!(
310        c,
311        // Usual ASCII suspects
312        '\u{0009}'   // \t
313        | '\u{000A}' // \n
314        | '\u{000B}' // vertical tab
315        | '\u{000C}' // form feed
316        | '\u{000D}' // \r
317        | '\u{0020}' // space
318
319        // NEXT LINE from latin1
320        | '\u{0085}'
321
322        // Bidi markers
323        | '\u{200E}' // LEFT-TO-RIGHT MARK
324        | '\u{200F}' // RIGHT-TO-LEFT MARK
325
326        // Dedicated whitespace characters from Unicode
327        | '\u{2028}' // LINE SEPARATOR
328        | '\u{2029}' // PARAGRAPH SEPARATOR
329    )
330}
331
332/// True if `c` is valid as a first character of an identifier.
333/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
334/// a formal definition of valid identifier name.
335pub fn is_id_start(c: char) -> bool {
336    // This is XID_Start OR '_' (which formally is not a XID_Start).
337    c == '_' || unicode_xid::UnicodeXID::is_xid_start(c)
338}
339
340/// True if `c` is valid as a non-first character of an identifier.
341/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
342/// a formal definition of valid identifier name.
343pub fn is_id_continue(c: char) -> bool {
344    unicode_xid::UnicodeXID::is_xid_continue(c)
345}
346
347/// The passed string is lexically an identifier.
348pub fn is_ident(string: &str) -> bool {
349    let mut chars = string.chars();
350    if let Some(start) = chars.next() {
351        is_id_start(start) && chars.all(is_id_continue)
352    } else {
353        false
354    }
355}
356
357impl Cursor<'_> {
358    /// Parses a token from the input string.
359    pub fn advance_token(&mut self) -> Token {
360        let first_char = match self.bump() {
361            Some(c) => c,
362            None => return Token::new(TokenKind::Eof, 0),
363        };
364        let token_kind = match first_char {
365            // Slash, comment or block comment.
366            '/' => match self.first() {
367                '/' => self.line_comment(),
368                '*' => self.block_comment(),
369                _ => Slash,
370            },
371
372            // Whitespace sequence.
373            c if is_whitespace(c) => self.whitespace(),
374
375            // Raw identifier, raw string literal or identifier.
376            'r' => match (self.first(), self.second()) {
377                ('#', c1) if is_id_start(c1) => self.raw_ident(),
378                ('#', _) | ('"', _) => {
379                    let res = self.raw_double_quoted_string(1);
380                    let suffix_start = self.pos_within_token();
381                    if res.is_ok() {
382                        self.eat_literal_suffix();
383                    }
384                    let kind = RawStr { n_hashes: res.ok() };
385                    Literal { kind, suffix_start }
386                }
387                _ => self.ident_or_unknown_prefix(),
388            },
389
390            // Byte literal, byte string literal, raw byte string literal or identifier.
391            'b' => self.c_or_byte_string(
392                |terminated| ByteStr { terminated },
393                |n_hashes| RawByteStr { n_hashes },
394                Some(|terminated| Byte { terminated }),
395            ),
396
397            // c-string literal, raw c-string literal or identifier.
398            'c' => self.c_or_byte_string(
399                |terminated| CStr { terminated },
400                |n_hashes| RawCStr { n_hashes },
401                None,
402            ),
403
404            // Identifier (this should be checked after other variant that can
405            // start as identifier).
406            c if is_id_start(c) => self.ident_or_unknown_prefix(),
407
408            // Numeric literal.
409            c @ '0'..='9' => {
410                let literal_kind = self.number(c);
411                let suffix_start = self.pos_within_token();
412                self.eat_literal_suffix();
413                TokenKind::Literal { kind: literal_kind, suffix_start }
414            }
415
416            // Guarded string literal prefix: `#"` or `##`
417            '#' if matches!(self.first(), '"' | '#') => {
418                self.bump();
419                TokenKind::GuardedStrPrefix
420            }
421
422            // One-symbol tokens.
423            ';' => Semi,
424            ',' => Comma,
425            '.' => Dot,
426            '(' => OpenParen,
427            ')' => CloseParen,
428            '{' => OpenBrace,
429            '}' => CloseBrace,
430            '[' => OpenBracket,
431            ']' => CloseBracket,
432            '@' => At,
433            '#' => Pound,
434            '~' => Tilde,
435            '?' => Question,
436            ':' => Colon,
437            '$' => Dollar,
438            '=' => Eq,
439            '!' => Bang,
440            '<' => Lt,
441            '>' => Gt,
442            '-' => Minus,
443            '&' => And,
444            '|' => Or,
445            '+' => Plus,
446            '*' => Star,
447            '^' => Caret,
448            '%' => Percent,
449
450            // Lifetime or character literal.
451            '\'' => self.lifetime_or_char(),
452
453            // String literal.
454            '"' => {
455                let terminated = self.double_quoted_string();
456                let suffix_start = self.pos_within_token();
457                if terminated {
458                    self.eat_literal_suffix();
459                }
460                let kind = Str { terminated };
461                Literal { kind, suffix_start }
462            }
463            // Identifier starting with an emoji. Only lexed for graceful error recovery.
464            c if !c.is_ascii() && c.is_emoji_char() => self.invalid_ident(),
465            _ => Unknown,
466        };
467        let res = Token::new(token_kind, self.pos_within_token());
468        self.reset_pos_within_token();
469        res
470    }
471
472    fn line_comment(&mut self) -> TokenKind {
473        debug_assert!(self.prev() == '/' && self.first() == '/');
474        self.bump();
475
476        let doc_style = match self.first() {
477            // `//!` is an inner line doc comment.
478            '!' => Some(DocStyle::Inner),
479            // `////` (more than 3 slashes) is not considered a doc comment.
480            '/' if self.second() != '/' => Some(DocStyle::Outer),
481            _ => None,
482        };
483
484        self.eat_until(b'\n');
485        LineComment { doc_style }
486    }
487
488    fn block_comment(&mut self) -> TokenKind {
489        debug_assert!(self.prev() == '/' && self.first() == '*');
490        self.bump();
491
492        let doc_style = match self.first() {
493            // `/*!` is an inner block doc comment.
494            '!' => Some(DocStyle::Inner),
495            // `/***` (more than 2 stars) is not considered a doc comment.
496            // `/**/` is not considered a doc comment.
497            '*' if !matches!(self.second(), '*' | '/') => Some(DocStyle::Outer),
498            _ => None,
499        };
500
501        let mut depth = 1usize;
502        while let Some(c) = self.bump() {
503            match c {
504                '/' if self.first() == '*' => {
505                    self.bump();
506                    depth += 1;
507                }
508                '*' if self.first() == '/' => {
509                    self.bump();
510                    depth -= 1;
511                    if depth == 0 {
512                        // This block comment is closed, so for a construction like "/* */ */"
513                        // there will be a successfully parsed block comment "/* */"
514                        // and " */" will be processed separately.
515                        break;
516                    }
517                }
518                _ => (),
519            }
520        }
521
522        BlockComment { doc_style, terminated: depth == 0 }
523    }
524
525    fn whitespace(&mut self) -> TokenKind {
526        debug_assert!(is_whitespace(self.prev()));
527        self.eat_while(is_whitespace);
528        Whitespace
529    }
530
531    fn raw_ident(&mut self) -> TokenKind {
532        debug_assert!(self.prev() == 'r' && self.first() == '#' && is_id_start(self.second()));
533        // Eat "#" symbol.
534        self.bump();
535        // Eat the identifier part of RawIdent.
536        self.eat_identifier();
537        RawIdent
538    }
539
540    fn ident_or_unknown_prefix(&mut self) -> TokenKind {
541        debug_assert!(is_id_start(self.prev()));
542        // Start is already eaten, eat the rest of identifier.
543        self.eat_while(is_id_continue);
544        // Known prefixes must have been handled earlier. So if
545        // we see a prefix here, it is definitely an unknown prefix.
546        match self.first() {
547            '#' | '"' | '\'' => UnknownPrefix,
548            c if !c.is_ascii() && c.is_emoji_char() => self.invalid_ident(),
549            _ => Ident,
550        }
551    }
552
553    fn invalid_ident(&mut self) -> TokenKind {
554        // Start is already eaten, eat the rest of identifier.
555        self.eat_while(|c| {
556            const ZERO_WIDTH_JOINER: char = '\u{200d}';
557            is_id_continue(c) || (!c.is_ascii() && c.is_emoji_char()) || c == ZERO_WIDTH_JOINER
558        });
559        // An invalid identifier followed by '#' or '"' or '\'' could be
560        // interpreted as an invalid literal prefix. We don't bother doing that
561        // because the treatment of invalid identifiers and invalid prefixes
562        // would be the same.
563        InvalidIdent
564    }
565
566    fn c_or_byte_string(
567        &mut self,
568        mk_kind: fn(bool) -> LiteralKind,
569        mk_kind_raw: fn(Option<u8>) -> LiteralKind,
570        single_quoted: Option<fn(bool) -> LiteralKind>,
571    ) -> TokenKind {
572        match (self.first(), self.second(), single_quoted) {
573            ('\'', _, Some(single_quoted)) => {
574                self.bump();
575                let terminated = self.single_quoted_string();
576                let suffix_start = self.pos_within_token();
577                if terminated {
578                    self.eat_literal_suffix();
579                }
580                let kind = single_quoted(terminated);
581                Literal { kind, suffix_start }
582            }
583            ('"', _, _) => {
584                self.bump();
585                let terminated = self.double_quoted_string();
586                let suffix_start = self.pos_within_token();
587                if terminated {
588                    self.eat_literal_suffix();
589                }
590                let kind = mk_kind(terminated);
591                Literal { kind, suffix_start }
592            }
593            ('r', '"', _) | ('r', '#', _) => {
594                self.bump();
595                let res = self.raw_double_quoted_string(2);
596                let suffix_start = self.pos_within_token();
597                if res.is_ok() {
598                    self.eat_literal_suffix();
599                }
600                let kind = mk_kind_raw(res.ok());
601                Literal { kind, suffix_start }
602            }
603            _ => self.ident_or_unknown_prefix(),
604        }
605    }
606
607    fn number(&mut self, first_digit: char) -> LiteralKind {
608        debug_assert!('0' <= self.prev() && self.prev() <= '9');
609        let mut base = Base::Decimal;
610        if first_digit == '0' {
611            // Attempt to parse encoding base.
612            match self.first() {
613                'b' => {
614                    base = Base::Binary;
615                    self.bump();
616                    if !self.eat_decimal_digits() {
617                        return Int { base, empty_int: true };
618                    }
619                }
620                'o' => {
621                    base = Base::Octal;
622                    self.bump();
623                    if !self.eat_decimal_digits() {
624                        return Int { base, empty_int: true };
625                    }
626                }
627                'x' => {
628                    base = Base::Hexadecimal;
629                    self.bump();
630                    if !self.eat_hexadecimal_digits() {
631                        return Int { base, empty_int: true };
632                    }
633                }
634                // Not a base prefix; consume additional digits.
635                '0'..='9' | '_' => {
636                    self.eat_decimal_digits();
637                }
638
639                // Also not a base prefix; nothing more to do here.
640                '.' | 'e' | 'E' => {}
641
642                // Just a 0.
643                _ => return Int { base, empty_int: false },
644            }
645        } else {
646            // No base prefix, parse number in the usual way.
647            self.eat_decimal_digits();
648        };
649
650        match self.first() {
651            // Don't be greedy if this is actually an
652            // integer literal followed by field/method access or a range pattern
653            // (`0..2` and `12.foo()`)
654            '.' if self.second() != '.' && !is_id_start(self.second()) => {
655                // might have stuff after the ., and if it does, it needs to start
656                // with a number
657                self.bump();
658                let mut empty_exponent = false;
659                if self.first().is_ascii_digit() {
660                    self.eat_decimal_digits();
661                    match self.first() {
662                        'e' | 'E' => {
663                            self.bump();
664                            empty_exponent = !self.eat_float_exponent();
665                        }
666                        _ => (),
667                    }
668                }
669                Float { base, empty_exponent }
670            }
671            'e' | 'E' => {
672                self.bump();
673                let empty_exponent = !self.eat_float_exponent();
674                Float { base, empty_exponent }
675            }
676            _ => Int { base, empty_int: false },
677        }
678    }
679
680    fn lifetime_or_char(&mut self) -> TokenKind {
681        debug_assert!(self.prev() == '\'');
682
683        let can_be_a_lifetime = if self.second() == '\'' {
684            // It's surely not a lifetime.
685            false
686        } else {
687            // If the first symbol is valid for identifier, it can be a lifetime.
688            // Also check if it's a number for a better error reporting (so '0 will
689            // be reported as invalid lifetime and not as unterminated char literal).
690            is_id_start(self.first()) || self.first().is_ascii_digit()
691        };
692
693        if !can_be_a_lifetime {
694            let terminated = self.single_quoted_string();
695            let suffix_start = self.pos_within_token();
696            if terminated {
697                self.eat_literal_suffix();
698            }
699            let kind = Char { terminated };
700            return Literal { kind, suffix_start };
701        }
702
703        if self.first() == 'r' && self.second() == '#' && is_id_start(self.third()) {
704            // Eat "r" and `#`, and identifier start characters.
705            self.bump();
706            self.bump();
707            self.bump();
708            self.eat_while(is_id_continue);
709            return RawLifetime;
710        }
711
712        // Either a lifetime or a character literal with
713        // length greater than 1.
714        let starts_with_number = self.first().is_ascii_digit();
715
716        // Skip the literal contents.
717        // First symbol can be a number (which isn't a valid identifier start),
718        // so skip it without any checks.
719        self.bump();
720        self.eat_while(is_id_continue);
721
722        match self.first() {
723            // Check if after skipping literal contents we've met a closing
724            // single quote (which means that user attempted to create a
725            // string with single quotes).
726            '\'' => {
727                self.bump();
728                let kind = Char { terminated: true };
729                Literal { kind, suffix_start: self.pos_within_token() }
730            }
731            '#' if !starts_with_number => UnknownPrefixLifetime,
732            _ => Lifetime { starts_with_number },
733        }
734    }
735
736    fn single_quoted_string(&mut self) -> bool {
737        debug_assert!(self.prev() == '\'');
738        // Check if it's a one-symbol literal.
739        if self.second() == '\'' && self.first() != '\\' {
740            self.bump();
741            self.bump();
742            return true;
743        }
744
745        // Literal has more than one symbol.
746
747        // Parse until either quotes are terminated or error is detected.
748        loop {
749            match self.first() {
750                // Quotes are terminated, finish parsing.
751                '\'' => {
752                    self.bump();
753                    return true;
754                }
755                // Probably beginning of the comment, which we don't want to include
756                // to the error report.
757                '/' => break,
758                // Newline without following '\'' means unclosed quote, stop parsing.
759                '\n' if self.second() != '\'' => break,
760                // End of file, stop parsing.
761                EOF_CHAR if self.is_eof() => break,
762                // Escaped slash is considered one character, so bump twice.
763                '\\' => {
764                    self.bump();
765                    self.bump();
766                }
767                // Skip the character.
768                _ => {
769                    self.bump();
770                }
771            }
772        }
773        // String was not terminated.
774        false
775    }
776
777    /// Eats double-quoted string and returns true
778    /// if string is terminated.
779    fn double_quoted_string(&mut self) -> bool {
780        debug_assert!(self.prev() == '"');
781        while let Some(c) = self.bump() {
782            match c {
783                '"' => {
784                    return true;
785                }
786                '\\' if self.first() == '\\' || self.first() == '"' => {
787                    // Bump again to skip escaped character.
788                    self.bump();
789                }
790                _ => (),
791            }
792        }
793        // End of file reached.
794        false
795    }
796
797    /// Attempt to lex for a guarded string literal.
798    ///
799    /// Used by `rustc_parse::lexer` to lex for guarded strings
800    /// conditionally based on edition.
801    ///
802    /// Note: this will not reset the `Cursor` when a
803    /// guarded string is not found. It is the caller's
804    /// responsibility to do so.
805    pub fn guarded_double_quoted_string(&mut self) -> Option<GuardedStr> {
806        debug_assert!(self.prev() != '#');
807
808        let mut n_start_hashes: u32 = 0;
809        while self.first() == '#' {
810            n_start_hashes += 1;
811            self.bump();
812        }
813
814        if self.first() != '"' {
815            return None;
816        }
817        self.bump();
818        debug_assert!(self.prev() == '"');
819
820        // Lex the string itself as a normal string literal
821        // so we can recover that for older editions later.
822        let terminated = self.double_quoted_string();
823        if !terminated {
824            let token_len = self.pos_within_token();
825            self.reset_pos_within_token();
826
827            return Some(GuardedStr { n_hashes: n_start_hashes, terminated: false, token_len });
828        }
829
830        // Consume closing '#' symbols.
831        // Note that this will not consume extra trailing `#` characters:
832        // `###"abcde"####` is lexed as a `GuardedStr { n_end_hashes: 3, .. }`
833        // followed by a `#` token.
834        let mut n_end_hashes = 0;
835        while self.first() == '#' && n_end_hashes < n_start_hashes {
836            n_end_hashes += 1;
837            self.bump();
838        }
839
840        // Reserved syntax, always an error, so it doesn't matter if
841        // `n_start_hashes != n_end_hashes`.
842
843        self.eat_literal_suffix();
844
845        let token_len = self.pos_within_token();
846        self.reset_pos_within_token();
847
848        Some(GuardedStr { n_hashes: n_start_hashes, terminated: true, token_len })
849    }
850
851    /// Eats the double-quoted string and returns `n_hashes` and an error if encountered.
852    fn raw_double_quoted_string(&mut self, prefix_len: u32) -> Result<u8, RawStrError> {
853        // Wrap the actual function to handle the error with too many hashes.
854        // This way, it eats the whole raw string.
855        let n_hashes = self.raw_string_unvalidated(prefix_len)?;
856        // Only up to 255 `#`s are allowed in raw strings
857        match u8::try_from(n_hashes) {
858            Ok(num) => Ok(num),
859            Err(_) => Err(RawStrError::TooManyDelimiters { found: n_hashes }),
860        }
861    }
862
863    fn raw_string_unvalidated(&mut self, prefix_len: u32) -> Result<u32, RawStrError> {
864        debug_assert!(self.prev() == 'r');
865        let start_pos = self.pos_within_token();
866        let mut possible_terminator_offset = None;
867        let mut max_hashes = 0;
868
869        // Count opening '#' symbols.
870        let mut eaten = 0;
871        while self.first() == '#' {
872            eaten += 1;
873            self.bump();
874        }
875        let n_start_hashes = eaten;
876
877        // Check that string is started.
878        match self.bump() {
879            Some('"') => (),
880            c => {
881                let c = c.unwrap_or(EOF_CHAR);
882                return Err(RawStrError::InvalidStarter { bad_char: c });
883            }
884        }
885
886        // Skip the string contents and on each '#' character met, check if this is
887        // a raw string termination.
888        loop {
889            self.eat_until(b'"');
890
891            if self.is_eof() {
892                return Err(RawStrError::NoTerminator {
893                    expected: n_start_hashes,
894                    found: max_hashes,
895                    possible_terminator_offset,
896                });
897            }
898
899            // Eat closing double quote.
900            self.bump();
901
902            // Check that amount of closing '#' symbols
903            // is equal to the amount of opening ones.
904            // Note that this will not consume extra trailing `#` characters:
905            // `r###"abcde"####` is lexed as a `RawStr { n_hashes: 3 }`
906            // followed by a `#` token.
907            let mut n_end_hashes = 0;
908            while self.first() == '#' && n_end_hashes < n_start_hashes {
909                n_end_hashes += 1;
910                self.bump();
911            }
912
913            if n_end_hashes == n_start_hashes {
914                return Ok(n_start_hashes);
915            } else if n_end_hashes > max_hashes {
916                // Keep track of possible terminators to give a hint about
917                // where there might be a missing terminator
918                possible_terminator_offset =
919                    Some(self.pos_within_token() - start_pos - n_end_hashes + prefix_len);
920                max_hashes = n_end_hashes;
921            }
922        }
923    }
924
925    fn eat_decimal_digits(&mut self) -> bool {
926        let mut has_digits = false;
927        loop {
928            match self.first() {
929                '_' => {
930                    self.bump();
931                }
932                '0'..='9' => {
933                    has_digits = true;
934                    self.bump();
935                }
936                _ => break,
937            }
938        }
939        has_digits
940    }
941
942    fn eat_hexadecimal_digits(&mut self) -> bool {
943        let mut has_digits = false;
944        loop {
945            match self.first() {
946                '_' => {
947                    self.bump();
948                }
949                '0'..='9' | 'a'..='f' | 'A'..='F' => {
950                    has_digits = true;
951                    self.bump();
952                }
953                _ => break,
954            }
955        }
956        has_digits
957    }
958
959    /// Eats the float exponent. Returns true if at least one digit was met,
960    /// and returns false otherwise.
961    fn eat_float_exponent(&mut self) -> bool {
962        debug_assert!(self.prev() == 'e' || self.prev() == 'E');
963        if self.first() == '-' || self.first() == '+' {
964            self.bump();
965        }
966        self.eat_decimal_digits()
967    }
968
969    // Eats the suffix of the literal, e.g. "u8".
970    fn eat_literal_suffix(&mut self) {
971        self.eat_identifier();
972    }
973
974    // Eats the identifier. Note: succeeds on `_`, which isn't a valid
975    // identifier.
976    fn eat_identifier(&mut self) {
977        if !is_id_start(self.first()) {
978            return;
979        }
980        self.bump();
981
982        self.eat_while(is_id_continue);
983    }
984}