rustc_lexer/lib.rs
1//! Low-level Rust lexer.
2//!
3//! The idea with `rustc_lexer` is to make a reusable library,
4//! by separating out pure lexing and rustc-specific concerns, like spans,
5//! error reporting, and interning. So, rustc_lexer operates directly on `&str`,
6//! produces simple tokens which are a pair of type-tag and a bit of original text,
7//! and does not report errors, instead storing them as flags on the token.
8//!
9//! Tokens produced by this lexer are not yet ready for parsing the Rust syntax.
10//! For that see [`rustc_parse::lexer`], which converts this basic token stream
11//! into wide tokens used by actual parser.
12//!
13//! The purpose of this crate is to convert raw sources into a labeled sequence
14//! of well-known token types, so building an actual Rust token stream will
15//! be easier.
16//!
17//! The main entity of this crate is the [`TokenKind`] enum which represents common
18//! lexeme types.
19//!
20//! [`rustc_parse::lexer`]: ../rustc_parse/lexer/index.html
21
22// tidy-alphabetical-start
23// We want to be able to build this crate with a stable compiler,
24// so no `#![feature]` attributes should be added.
25#![deny(unstable_features)]
26// tidy-alphabetical-end
27
28mod cursor;
29pub mod unescape;
30
31#[cfg(test)]
32mod tests;
33
34use unicode_properties::UnicodeEmoji;
35pub use unicode_xid::UNICODE_VERSION as UNICODE_XID_VERSION;
36
37use self::LiteralKind::*;
38use self::TokenKind::*;
39pub use crate::cursor::Cursor;
40use crate::cursor::EOF_CHAR;
41
42/// Parsed token.
43/// It doesn't contain information about data that has been parsed,
44/// only the type of the token and its size.
45#[derive(Debug)]
46pub struct Token {
47 pub kind: TokenKind,
48 pub len: u32,
49}
50
51impl Token {
52 fn new(kind: TokenKind, len: u32) -> Token {
53 Token { kind, len }
54 }
55}
56
57/// Enum representing common lexeme types.
58#[derive(Clone, Copy, Debug, PartialEq, Eq)]
59pub enum TokenKind {
60 /// A line comment, e.g. `// comment`.
61 LineComment { doc_style: Option<DocStyle> },
62
63 /// A block comment, e.g. `/* block comment */`.
64 ///
65 /// Block comments can be recursive, so a sequence like `/* /* */`
66 /// will not be considered terminated and will result in a parsing error.
67 BlockComment { doc_style: Option<DocStyle>, terminated: bool },
68
69 /// Any whitespace character sequence.
70 Whitespace,
71
72 /// An identifier or keyword, e.g. `ident` or `continue`.
73 Ident,
74
75 /// An identifier that is invalid because it contains emoji.
76 InvalidIdent,
77
78 /// A raw identifier, e.g. "r#ident".
79 RawIdent,
80
81 /// An unknown literal prefix, like `foo#`, `foo'`, `foo"`. Excludes
82 /// literal prefixes that contain emoji, which are considered "invalid".
83 ///
84 /// Note that only the
85 /// prefix (`foo`) is included in the token, not the separator (which is
86 /// lexed as its own distinct token). In Rust 2021 and later, reserved
87 /// prefixes are reported as errors; in earlier editions, they result in a
88 /// (allowed by default) lint, and are treated as regular identifier
89 /// tokens.
90 UnknownPrefix,
91
92 /// An unknown prefix in a lifetime, like `'foo#`.
93 ///
94 /// Like `UnknownPrefix`, only the `'` and prefix are included in the token
95 /// and not the separator.
96 UnknownPrefixLifetime,
97
98 /// A raw lifetime, e.g. `'r#foo`. In edition < 2021 it will be split into
99 /// several tokens: `'r` and `#` and `foo`.
100 RawLifetime,
101
102 /// Guarded string literal prefix: `#"` or `##`.
103 ///
104 /// Used for reserving "guarded strings" (RFC 3598) in edition 2024.
105 /// Split into the component tokens on older editions.
106 GuardedStrPrefix,
107
108 /// Literals, e.g. `12u8`, `1.0e-40`, `b"123"`. Note that `_` is an invalid
109 /// suffix, but may be present here on string and float literals. Users of
110 /// this type will need to check for and reject that case.
111 ///
112 /// See [LiteralKind] for more details.
113 Literal { kind: LiteralKind, suffix_start: u32 },
114
115 /// A lifetime, e.g. `'a`.
116 Lifetime { starts_with_number: bool },
117
118 /// `;`
119 Semi,
120 /// `,`
121 Comma,
122 /// `.`
123 Dot,
124 /// `(`
125 OpenParen,
126 /// `)`
127 CloseParen,
128 /// `{`
129 OpenBrace,
130 /// `}`
131 CloseBrace,
132 /// `[`
133 OpenBracket,
134 /// `]`
135 CloseBracket,
136 /// `@`
137 At,
138 /// `#`
139 Pound,
140 /// `~`
141 Tilde,
142 /// `?`
143 Question,
144 /// `:`
145 Colon,
146 /// `$`
147 Dollar,
148 /// `=`
149 Eq,
150 /// `!`
151 Bang,
152 /// `<`
153 Lt,
154 /// `>`
155 Gt,
156 /// `-`
157 Minus,
158 /// `&`
159 And,
160 /// `|`
161 Or,
162 /// `+`
163 Plus,
164 /// `*`
165 Star,
166 /// `/`
167 Slash,
168 /// `^`
169 Caret,
170 /// `%`
171 Percent,
172
173 /// Unknown token, not expected by the lexer, e.g. "№"
174 Unknown,
175
176 /// End of input.
177 Eof,
178}
179
180#[derive(Clone, Copy, Debug, PartialEq, Eq)]
181pub enum DocStyle {
182 Outer,
183 Inner,
184}
185
186/// Enum representing the literal types supported by the lexer.
187///
188/// Note that the suffix is *not* considered when deciding the `LiteralKind` in
189/// this type. This means that float literals like `1f32` are classified by this
190/// type as `Int`. (Compare against `rustc_ast::token::LitKind` and
191/// `rustc_ast::ast::LitKind`).
192#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
193pub enum LiteralKind {
194 /// `12_u8`, `0o100`, `0b120i99`, `1f32`.
195 Int { base: Base, empty_int: bool },
196 /// `12.34f32`, `1e3`, but not `1f32`.
197 Float { base: Base, empty_exponent: bool },
198 /// `'a'`, `'\\'`, `'''`, `';`
199 Char { terminated: bool },
200 /// `b'a'`, `b'\\'`, `b'''`, `b';`
201 Byte { terminated: bool },
202 /// `"abc"`, `"abc`
203 Str { terminated: bool },
204 /// `b"abc"`, `b"abc`
205 ByteStr { terminated: bool },
206 /// `c"abc"`, `c"abc`
207 CStr { terminated: bool },
208 /// `r"abc"`, `r#"abc"#`, `r####"ab"###"c"####`, `r#"a`. `None` indicates
209 /// an invalid literal.
210 RawStr { n_hashes: Option<u8> },
211 /// `br"abc"`, `br#"abc"#`, `br####"ab"###"c"####`, `br#"a`. `None`
212 /// indicates an invalid literal.
213 RawByteStr { n_hashes: Option<u8> },
214 /// `cr"abc"`, "cr#"abc"#", `cr#"a`. `None` indicates an invalid literal.
215 RawCStr { n_hashes: Option<u8> },
216}
217
218/// `#"abc"#`, `##"a"` (fewer closing), or even `#"a` (unterminated).
219///
220/// Can capture fewer closing hashes than starting hashes,
221/// for more efficient lexing and better backwards diagnostics.
222#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
223pub struct GuardedStr {
224 pub n_hashes: u32,
225 pub terminated: bool,
226 pub token_len: u32,
227}
228
229#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
230pub enum RawStrError {
231 /// Non `#` characters exist between `r` and `"`, e.g. `r##~"abcde"##`
232 InvalidStarter { bad_char: char },
233 /// The string was not terminated, e.g. `r###"abcde"##`.
234 /// `possible_terminator_offset` is the number of characters after `r` or
235 /// `br` where they may have intended to terminate it.
236 NoTerminator { expected: u32, found: u32, possible_terminator_offset: Option<u32> },
237 /// More than 255 `#`s exist.
238 TooManyDelimiters { found: u32 },
239}
240
241/// Base of numeric literal encoding according to its prefix.
242#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
243pub enum Base {
244 /// Literal starts with "0b".
245 Binary = 2,
246 /// Literal starts with "0o".
247 Octal = 8,
248 /// Literal doesn't contain a prefix.
249 Decimal = 10,
250 /// Literal starts with "0x".
251 Hexadecimal = 16,
252}
253
254/// `rustc` allows files to have a shebang, e.g. "#!/usr/bin/rustrun",
255/// but shebang isn't a part of rust syntax.
256pub fn strip_shebang(input: &str) -> Option<usize> {
257 // Shebang must start with `#!` literally, without any preceding whitespace.
258 // For simplicity we consider any line starting with `#!` a shebang,
259 // regardless of restrictions put on shebangs by specific platforms.
260 if let Some(input_tail) = input.strip_prefix("#!") {
261 // Ok, this is a shebang but if the next non-whitespace token is `[`,
262 // then it may be valid Rust code, so consider it Rust code.
263 let next_non_whitespace_token = tokenize(input_tail).map(|tok| tok.kind).find(|tok| {
264 !matches!(
265 tok,
266 TokenKind::Whitespace
267 | TokenKind::LineComment { doc_style: None }
268 | TokenKind::BlockComment { doc_style: None, .. }
269 )
270 });
271 if next_non_whitespace_token != Some(TokenKind::OpenBracket) {
272 // No other choice than to consider this a shebang.
273 return Some(2 + input_tail.lines().next().unwrap_or_default().len());
274 }
275 }
276 None
277}
278
279/// Validates a raw string literal. Used for getting more information about a
280/// problem with a `RawStr`/`RawByteStr` with a `None` field.
281#[inline]
282pub fn validate_raw_str(input: &str, prefix_len: u32) -> Result<(), RawStrError> {
283 debug_assert!(!input.is_empty());
284 let mut cursor = Cursor::new(input);
285 // Move past the leading `r` or `br`.
286 for _ in 0..prefix_len {
287 cursor.bump().unwrap();
288 }
289 cursor.raw_double_quoted_string(prefix_len).map(|_| ())
290}
291
292/// Creates an iterator that produces tokens from the input string.
293pub fn tokenize(input: &str) -> impl Iterator<Item = Token> {
294 let mut cursor = Cursor::new(input);
295 std::iter::from_fn(move || {
296 let token = cursor.advance_token();
297 if token.kind != TokenKind::Eof { Some(token) } else { None }
298 })
299}
300
301/// True if `c` is considered a whitespace according to Rust language definition.
302/// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
303/// for definitions of these classes.
304pub fn is_whitespace(c: char) -> bool {
305 // This is Pattern_White_Space.
306 //
307 // Note that this set is stable (ie, it doesn't change with different
308 // Unicode versions), so it's ok to just hard-code the values.
309
310 matches!(
311 c,
312 // Usual ASCII suspects
313 '\u{0009}' // \t
314 | '\u{000A}' // \n
315 | '\u{000B}' // vertical tab
316 | '\u{000C}' // form feed
317 | '\u{000D}' // \r
318 | '\u{0020}' // space
319
320 // NEXT LINE from latin1
321 | '\u{0085}'
322
323 // Bidi markers
324 | '\u{200E}' // LEFT-TO-RIGHT MARK
325 | '\u{200F}' // RIGHT-TO-LEFT MARK
326
327 // Dedicated whitespace characters from Unicode
328 | '\u{2028}' // LINE SEPARATOR
329 | '\u{2029}' // PARAGRAPH SEPARATOR
330 )
331}
332
333/// True if `c` is valid as a first character of an identifier.
334/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
335/// a formal definition of valid identifier name.
336pub fn is_id_start(c: char) -> bool {
337 // This is XID_Start OR '_' (which formally is not a XID_Start).
338 c == '_' || unicode_xid::UnicodeXID::is_xid_start(c)
339}
340
341/// True if `c` is valid as a non-first character of an identifier.
342/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
343/// a formal definition of valid identifier name.
344pub fn is_id_continue(c: char) -> bool {
345 unicode_xid::UnicodeXID::is_xid_continue(c)
346}
347
348/// The passed string is lexically an identifier.
349pub fn is_ident(string: &str) -> bool {
350 let mut chars = string.chars();
351 if let Some(start) = chars.next() {
352 is_id_start(start) && chars.all(is_id_continue)
353 } else {
354 false
355 }
356}
357
358impl Cursor<'_> {
359 /// Parses a token from the input string.
360 pub fn advance_token(&mut self) -> Token {
361 let first_char = match self.bump() {
362 Some(c) => c,
363 None => return Token::new(TokenKind::Eof, 0),
364 };
365 let token_kind = match first_char {
366 // Slash, comment or block comment.
367 '/' => match self.first() {
368 '/' => self.line_comment(),
369 '*' => self.block_comment(),
370 _ => Slash,
371 },
372
373 // Whitespace sequence.
374 c if is_whitespace(c) => self.whitespace(),
375
376 // Raw identifier, raw string literal or identifier.
377 'r' => match (self.first(), self.second()) {
378 ('#', c1) if is_id_start(c1) => self.raw_ident(),
379 ('#', _) | ('"', _) => {
380 let res = self.raw_double_quoted_string(1);
381 let suffix_start = self.pos_within_token();
382 if res.is_ok() {
383 self.eat_literal_suffix();
384 }
385 let kind = RawStr { n_hashes: res.ok() };
386 Literal { kind, suffix_start }
387 }
388 _ => self.ident_or_unknown_prefix(),
389 },
390
391 // Byte literal, byte string literal, raw byte string literal or identifier.
392 'b' => self.c_or_byte_string(
393 |terminated| ByteStr { terminated },
394 |n_hashes| RawByteStr { n_hashes },
395 Some(|terminated| Byte { terminated }),
396 ),
397
398 // c-string literal, raw c-string literal or identifier.
399 'c' => self.c_or_byte_string(
400 |terminated| CStr { terminated },
401 |n_hashes| RawCStr { n_hashes },
402 None,
403 ),
404
405 // Identifier (this should be checked after other variant that can
406 // start as identifier).
407 c if is_id_start(c) => self.ident_or_unknown_prefix(),
408
409 // Numeric literal.
410 c @ '0'..='9' => {
411 let literal_kind = self.number(c);
412 let suffix_start = self.pos_within_token();
413 self.eat_literal_suffix();
414 TokenKind::Literal { kind: literal_kind, suffix_start }
415 }
416
417 // Guarded string literal prefix: `#"` or `##`
418 '#' if matches!(self.first(), '"' | '#') => {
419 self.bump();
420 TokenKind::GuardedStrPrefix
421 }
422
423 // One-symbol tokens.
424 ';' => Semi,
425 ',' => Comma,
426 '.' => Dot,
427 '(' => OpenParen,
428 ')' => CloseParen,
429 '{' => OpenBrace,
430 '}' => CloseBrace,
431 '[' => OpenBracket,
432 ']' => CloseBracket,
433 '@' => At,
434 '#' => Pound,
435 '~' => Tilde,
436 '?' => Question,
437 ':' => Colon,
438 '$' => Dollar,
439 '=' => Eq,
440 '!' => Bang,
441 '<' => Lt,
442 '>' => Gt,
443 '-' => Minus,
444 '&' => And,
445 '|' => Or,
446 '+' => Plus,
447 '*' => Star,
448 '^' => Caret,
449 '%' => Percent,
450
451 // Lifetime or character literal.
452 '\'' => self.lifetime_or_char(),
453
454 // String literal.
455 '"' => {
456 let terminated = self.double_quoted_string();
457 let suffix_start = self.pos_within_token();
458 if terminated {
459 self.eat_literal_suffix();
460 }
461 let kind = Str { terminated };
462 Literal { kind, suffix_start }
463 }
464 // Identifier starting with an emoji. Only lexed for graceful error recovery.
465 c if !c.is_ascii() && c.is_emoji_char() => self.invalid_ident(),
466 _ => Unknown,
467 };
468 let res = Token::new(token_kind, self.pos_within_token());
469 self.reset_pos_within_token();
470 res
471 }
472
473 fn line_comment(&mut self) -> TokenKind {
474 debug_assert!(self.prev() == '/' && self.first() == '/');
475 self.bump();
476
477 let doc_style = match self.first() {
478 // `//!` is an inner line doc comment.
479 '!' => Some(DocStyle::Inner),
480 // `////` (more than 3 slashes) is not considered a doc comment.
481 '/' if self.second() != '/' => Some(DocStyle::Outer),
482 _ => None,
483 };
484
485 self.eat_until(b'\n');
486 LineComment { doc_style }
487 }
488
489 fn block_comment(&mut self) -> TokenKind {
490 debug_assert!(self.prev() == '/' && self.first() == '*');
491 self.bump();
492
493 let doc_style = match self.first() {
494 // `/*!` is an inner block doc comment.
495 '!' => Some(DocStyle::Inner),
496 // `/***` (more than 2 stars) is not considered a doc comment.
497 // `/**/` is not considered a doc comment.
498 '*' if !matches!(self.second(), '*' | '/') => Some(DocStyle::Outer),
499 _ => None,
500 };
501
502 let mut depth = 1usize;
503 while let Some(c) = self.bump() {
504 match c {
505 '/' if self.first() == '*' => {
506 self.bump();
507 depth += 1;
508 }
509 '*' if self.first() == '/' => {
510 self.bump();
511 depth -= 1;
512 if depth == 0 {
513 // This block comment is closed, so for a construction like "/* */ */"
514 // there will be a successfully parsed block comment "/* */"
515 // and " */" will be processed separately.
516 break;
517 }
518 }
519 _ => (),
520 }
521 }
522
523 BlockComment { doc_style, terminated: depth == 0 }
524 }
525
526 fn whitespace(&mut self) -> TokenKind {
527 debug_assert!(is_whitespace(self.prev()));
528 self.eat_while(is_whitespace);
529 Whitespace
530 }
531
532 fn raw_ident(&mut self) -> TokenKind {
533 debug_assert!(self.prev() == 'r' && self.first() == '#' && is_id_start(self.second()));
534 // Eat "#" symbol.
535 self.bump();
536 // Eat the identifier part of RawIdent.
537 self.eat_identifier();
538 RawIdent
539 }
540
541 fn ident_or_unknown_prefix(&mut self) -> TokenKind {
542 debug_assert!(is_id_start(self.prev()));
543 // Start is already eaten, eat the rest of identifier.
544 self.eat_while(is_id_continue);
545 // Known prefixes must have been handled earlier. So if
546 // we see a prefix here, it is definitely an unknown prefix.
547 match self.first() {
548 '#' | '"' | '\'' => UnknownPrefix,
549 c if !c.is_ascii() && c.is_emoji_char() => self.invalid_ident(),
550 _ => Ident,
551 }
552 }
553
554 fn invalid_ident(&mut self) -> TokenKind {
555 // Start is already eaten, eat the rest of identifier.
556 self.eat_while(|c| {
557 const ZERO_WIDTH_JOINER: char = '\u{200d}';
558 is_id_continue(c) || (!c.is_ascii() && c.is_emoji_char()) || c == ZERO_WIDTH_JOINER
559 });
560 // An invalid identifier followed by '#' or '"' or '\'' could be
561 // interpreted as an invalid literal prefix. We don't bother doing that
562 // because the treatment of invalid identifiers and invalid prefixes
563 // would be the same.
564 InvalidIdent
565 }
566
567 fn c_or_byte_string(
568 &mut self,
569 mk_kind: fn(bool) -> LiteralKind,
570 mk_kind_raw: fn(Option<u8>) -> LiteralKind,
571 single_quoted: Option<fn(bool) -> LiteralKind>,
572 ) -> TokenKind {
573 match (self.first(), self.second(), single_quoted) {
574 ('\'', _, Some(single_quoted)) => {
575 self.bump();
576 let terminated = self.single_quoted_string();
577 let suffix_start = self.pos_within_token();
578 if terminated {
579 self.eat_literal_suffix();
580 }
581 let kind = single_quoted(terminated);
582 Literal { kind, suffix_start }
583 }
584 ('"', _, _) => {
585 self.bump();
586 let terminated = self.double_quoted_string();
587 let suffix_start = self.pos_within_token();
588 if terminated {
589 self.eat_literal_suffix();
590 }
591 let kind = mk_kind(terminated);
592 Literal { kind, suffix_start }
593 }
594 ('r', '"', _) | ('r', '#', _) => {
595 self.bump();
596 let res = self.raw_double_quoted_string(2);
597 let suffix_start = self.pos_within_token();
598 if res.is_ok() {
599 self.eat_literal_suffix();
600 }
601 let kind = mk_kind_raw(res.ok());
602 Literal { kind, suffix_start }
603 }
604 _ => self.ident_or_unknown_prefix(),
605 }
606 }
607
608 fn number(&mut self, first_digit: char) -> LiteralKind {
609 debug_assert!('0' <= self.prev() && self.prev() <= '9');
610 let mut base = Base::Decimal;
611 if first_digit == '0' {
612 // Attempt to parse encoding base.
613 match self.first() {
614 'b' => {
615 base = Base::Binary;
616 self.bump();
617 if !self.eat_decimal_digits() {
618 return Int { base, empty_int: true };
619 }
620 }
621 'o' => {
622 base = Base::Octal;
623 self.bump();
624 if !self.eat_decimal_digits() {
625 return Int { base, empty_int: true };
626 }
627 }
628 'x' => {
629 base = Base::Hexadecimal;
630 self.bump();
631 if !self.eat_hexadecimal_digits() {
632 return Int { base, empty_int: true };
633 }
634 }
635 // Not a base prefix; consume additional digits.
636 '0'..='9' | '_' => {
637 self.eat_decimal_digits();
638 }
639
640 // Also not a base prefix; nothing more to do here.
641 '.' | 'e' | 'E' => {}
642
643 // Just a 0.
644 _ => return Int { base, empty_int: false },
645 }
646 } else {
647 // No base prefix, parse number in the usual way.
648 self.eat_decimal_digits();
649 };
650
651 match self.first() {
652 // Don't be greedy if this is actually an
653 // integer literal followed by field/method access or a range pattern
654 // (`0..2` and `12.foo()`)
655 '.' if self.second() != '.' && !is_id_start(self.second()) => {
656 // might have stuff after the ., and if it does, it needs to start
657 // with a number
658 self.bump();
659 let mut empty_exponent = false;
660 if self.first().is_ascii_digit() {
661 self.eat_decimal_digits();
662 match self.first() {
663 'e' | 'E' => {
664 self.bump();
665 empty_exponent = !self.eat_float_exponent();
666 }
667 _ => (),
668 }
669 }
670 Float { base, empty_exponent }
671 }
672 'e' | 'E' => {
673 self.bump();
674 let empty_exponent = !self.eat_float_exponent();
675 Float { base, empty_exponent }
676 }
677 _ => Int { base, empty_int: false },
678 }
679 }
680
681 fn lifetime_or_char(&mut self) -> TokenKind {
682 debug_assert!(self.prev() == '\'');
683
684 let can_be_a_lifetime = if self.second() == '\'' {
685 // It's surely not a lifetime.
686 false
687 } else {
688 // If the first symbol is valid for identifier, it can be a lifetime.
689 // Also check if it's a number for a better error reporting (so '0 will
690 // be reported as invalid lifetime and not as unterminated char literal).
691 is_id_start(self.first()) || self.first().is_ascii_digit()
692 };
693
694 if !can_be_a_lifetime {
695 let terminated = self.single_quoted_string();
696 let suffix_start = self.pos_within_token();
697 if terminated {
698 self.eat_literal_suffix();
699 }
700 let kind = Char { terminated };
701 return Literal { kind, suffix_start };
702 }
703
704 if self.first() == 'r' && self.second() == '#' && is_id_start(self.third()) {
705 // Eat "r" and `#`, and identifier start characters.
706 self.bump();
707 self.bump();
708 self.bump();
709 self.eat_while(is_id_continue);
710 return RawLifetime;
711 }
712
713 // Either a lifetime or a character literal with
714 // length greater than 1.
715 let starts_with_number = self.first().is_ascii_digit();
716
717 // Skip the literal contents.
718 // First symbol can be a number (which isn't a valid identifier start),
719 // so skip it without any checks.
720 self.bump();
721 self.eat_while(is_id_continue);
722
723 match self.first() {
724 // Check if after skipping literal contents we've met a closing
725 // single quote (which means that user attempted to create a
726 // string with single quotes).
727 '\'' => {
728 self.bump();
729 let kind = Char { terminated: true };
730 Literal { kind, suffix_start: self.pos_within_token() }
731 }
732 '#' if !starts_with_number => UnknownPrefixLifetime,
733 _ => Lifetime { starts_with_number },
734 }
735 }
736
737 fn single_quoted_string(&mut self) -> bool {
738 debug_assert!(self.prev() == '\'');
739 // Check if it's a one-symbol literal.
740 if self.second() == '\'' && self.first() != '\\' {
741 self.bump();
742 self.bump();
743 return true;
744 }
745
746 // Literal has more than one symbol.
747
748 // Parse until either quotes are terminated or error is detected.
749 loop {
750 match self.first() {
751 // Quotes are terminated, finish parsing.
752 '\'' => {
753 self.bump();
754 return true;
755 }
756 // Probably beginning of the comment, which we don't want to include
757 // to the error report.
758 '/' => break,
759 // Newline without following '\'' means unclosed quote, stop parsing.
760 '\n' if self.second() != '\'' => break,
761 // End of file, stop parsing.
762 EOF_CHAR if self.is_eof() => break,
763 // Escaped slash is considered one character, so bump twice.
764 '\\' => {
765 self.bump();
766 self.bump();
767 }
768 // Skip the character.
769 _ => {
770 self.bump();
771 }
772 }
773 }
774 // String was not terminated.
775 false
776 }
777
778 /// Eats double-quoted string and returns true
779 /// if string is terminated.
780 fn double_quoted_string(&mut self) -> bool {
781 debug_assert!(self.prev() == '"');
782 while let Some(c) = self.bump() {
783 match c {
784 '"' => {
785 return true;
786 }
787 '\\' if self.first() == '\\' || self.first() == '"' => {
788 // Bump again to skip escaped character.
789 self.bump();
790 }
791 _ => (),
792 }
793 }
794 // End of file reached.
795 false
796 }
797
798 /// Attempt to lex for a guarded string literal.
799 ///
800 /// Used by `rustc_parse::lexer` to lex for guarded strings
801 /// conditionally based on edition.
802 ///
803 /// Note: this will not reset the `Cursor` when a
804 /// guarded string is not found. It is the caller's
805 /// responsibility to do so.
806 pub fn guarded_double_quoted_string(&mut self) -> Option<GuardedStr> {
807 debug_assert!(self.prev() != '#');
808
809 let mut n_start_hashes: u32 = 0;
810 while self.first() == '#' {
811 n_start_hashes += 1;
812 self.bump();
813 }
814
815 if self.first() != '"' {
816 return None;
817 }
818 self.bump();
819 debug_assert!(self.prev() == '"');
820
821 // Lex the string itself as a normal string literal
822 // so we can recover that for older editions later.
823 let terminated = self.double_quoted_string();
824 if !terminated {
825 let token_len = self.pos_within_token();
826 self.reset_pos_within_token();
827
828 return Some(GuardedStr { n_hashes: n_start_hashes, terminated: false, token_len });
829 }
830
831 // Consume closing '#' symbols.
832 // Note that this will not consume extra trailing `#` characters:
833 // `###"abcde"####` is lexed as a `GuardedStr { n_end_hashes: 3, .. }`
834 // followed by a `#` token.
835 let mut n_end_hashes = 0;
836 while self.first() == '#' && n_end_hashes < n_start_hashes {
837 n_end_hashes += 1;
838 self.bump();
839 }
840
841 // Reserved syntax, always an error, so it doesn't matter if
842 // `n_start_hashes != n_end_hashes`.
843
844 self.eat_literal_suffix();
845
846 let token_len = self.pos_within_token();
847 self.reset_pos_within_token();
848
849 Some(GuardedStr { n_hashes: n_start_hashes, terminated: true, token_len })
850 }
851
852 /// Eats the double-quoted string and returns `n_hashes` and an error if encountered.
853 fn raw_double_quoted_string(&mut self, prefix_len: u32) -> Result<u8, RawStrError> {
854 // Wrap the actual function to handle the error with too many hashes.
855 // This way, it eats the whole raw string.
856 let n_hashes = self.raw_string_unvalidated(prefix_len)?;
857 // Only up to 255 `#`s are allowed in raw strings
858 match u8::try_from(n_hashes) {
859 Ok(num) => Ok(num),
860 Err(_) => Err(RawStrError::TooManyDelimiters { found: n_hashes }),
861 }
862 }
863
864 fn raw_string_unvalidated(&mut self, prefix_len: u32) -> Result<u32, RawStrError> {
865 debug_assert!(self.prev() == 'r');
866 let start_pos = self.pos_within_token();
867 let mut possible_terminator_offset = None;
868 let mut max_hashes = 0;
869
870 // Count opening '#' symbols.
871 let mut eaten = 0;
872 while self.first() == '#' {
873 eaten += 1;
874 self.bump();
875 }
876 let n_start_hashes = eaten;
877
878 // Check that string is started.
879 match self.bump() {
880 Some('"') => (),
881 c => {
882 let c = c.unwrap_or(EOF_CHAR);
883 return Err(RawStrError::InvalidStarter { bad_char: c });
884 }
885 }
886
887 // Skip the string contents and on each '#' character met, check if this is
888 // a raw string termination.
889 loop {
890 self.eat_until(b'"');
891
892 if self.is_eof() {
893 return Err(RawStrError::NoTerminator {
894 expected: n_start_hashes,
895 found: max_hashes,
896 possible_terminator_offset,
897 });
898 }
899
900 // Eat closing double quote.
901 self.bump();
902
903 // Check that amount of closing '#' symbols
904 // is equal to the amount of opening ones.
905 // Note that this will not consume extra trailing `#` characters:
906 // `r###"abcde"####` is lexed as a `RawStr { n_hashes: 3 }`
907 // followed by a `#` token.
908 let mut n_end_hashes = 0;
909 while self.first() == '#' && n_end_hashes < n_start_hashes {
910 n_end_hashes += 1;
911 self.bump();
912 }
913
914 if n_end_hashes == n_start_hashes {
915 return Ok(n_start_hashes);
916 } else if n_end_hashes > max_hashes {
917 // Keep track of possible terminators to give a hint about
918 // where there might be a missing terminator
919 possible_terminator_offset =
920 Some(self.pos_within_token() - start_pos - n_end_hashes + prefix_len);
921 max_hashes = n_end_hashes;
922 }
923 }
924 }
925
926 fn eat_decimal_digits(&mut self) -> bool {
927 let mut has_digits = false;
928 loop {
929 match self.first() {
930 '_' => {
931 self.bump();
932 }
933 '0'..='9' => {
934 has_digits = true;
935 self.bump();
936 }
937 _ => break,
938 }
939 }
940 has_digits
941 }
942
943 fn eat_hexadecimal_digits(&mut self) -> bool {
944 let mut has_digits = false;
945 loop {
946 match self.first() {
947 '_' => {
948 self.bump();
949 }
950 '0'..='9' | 'a'..='f' | 'A'..='F' => {
951 has_digits = true;
952 self.bump();
953 }
954 _ => break,
955 }
956 }
957 has_digits
958 }
959
960 /// Eats the float exponent. Returns true if at least one digit was met,
961 /// and returns false otherwise.
962 fn eat_float_exponent(&mut self) -> bool {
963 debug_assert!(self.prev() == 'e' || self.prev() == 'E');
964 if self.first() == '-' || self.first() == '+' {
965 self.bump();
966 }
967 self.eat_decimal_digits()
968 }
969
970 // Eats the suffix of the literal, e.g. "u8".
971 fn eat_literal_suffix(&mut self) {
972 self.eat_identifier();
973 }
974
975 // Eats the identifier. Note: succeeds on `_`, which isn't a valid
976 // identifier.
977 fn eat_identifier(&mut self) {
978 if !is_id_start(self.first()) {
979 return;
980 }
981 self.bump();
982
983 self.eat_while(is_id_continue);
984 }
985}