rustc_lexer/lib.rs
1//! Low-level Rust lexer.
2//!
3//! The idea with `rustc_lexer` is to make a reusable library,
4//! by separating out pure lexing and rustc-specific concerns, like spans,
5//! error reporting, and interning. So, rustc_lexer operates directly on `&str`,
6//! produces simple tokens which are a pair of type-tag and a bit of original text,
7//! and does not report errors, instead storing them as flags on the token.
8//!
9//! Tokens produced by this lexer are not yet ready for parsing the Rust syntax.
10//! For that see [`rustc_parse::lexer`], which converts this basic token stream
11//! into wide tokens used by actual parser.
12//!
13//! The purpose of this crate is to convert raw sources into a labeled sequence
14//! of well-known token types, so building an actual Rust token stream will
15//! be easier.
16//!
17//! The main entity of this crate is the [`TokenKind`] enum which represents common
18//! lexeme types.
19//!
20//! [`rustc_parse::lexer`]: ../rustc_parse/lexer/index.html
21
22// tidy-alphabetical-start
23// We want to be able to build this crate with a stable compiler,
24// so no `#![feature]` attributes should be added.
25#![deny(unstable_features)]
26// tidy-alphabetical-end
27
28mod cursor;
29
30#[cfg(test)]
31mod tests;
32
33use unicode_properties::UnicodeEmoji;
34pub use unicode_xid::UNICODE_VERSION as UNICODE_XID_VERSION;
35
36use self::LiteralKind::*;
37use self::TokenKind::*;
38pub use crate::cursor::Cursor;
39use crate::cursor::EOF_CHAR;
40
41/// Parsed token.
42/// It doesn't contain information about data that has been parsed,
43/// only the type of the token and its size.
44#[derive(Debug)]
45pub struct Token {
46 pub kind: TokenKind,
47 pub len: u32,
48}
49
50impl Token {
51 fn new(kind: TokenKind, len: u32) -> Token {
52 Token { kind, len }
53 }
54}
55
56/// Enum representing common lexeme types.
57#[derive(Clone, Copy, Debug, PartialEq, Eq)]
58pub enum TokenKind {
59 /// A line comment, e.g. `// comment`.
60 LineComment { doc_style: Option<DocStyle> },
61
62 /// A block comment, e.g. `/* block comment */`.
63 ///
64 /// Block comments can be recursive, so a sequence like `/* /* */`
65 /// will not be considered terminated and will result in a parsing error.
66 BlockComment { doc_style: Option<DocStyle>, terminated: bool },
67
68 /// Any whitespace character sequence.
69 Whitespace,
70
71 /// An identifier or keyword, e.g. `ident` or `continue`.
72 Ident,
73
74 /// An identifier that is invalid because it contains emoji.
75 InvalidIdent,
76
77 /// A raw identifier, e.g. "r#ident".
78 RawIdent,
79
80 /// An unknown literal prefix, like `foo#`, `foo'`, `foo"`. Excludes
81 /// literal prefixes that contain emoji, which are considered "invalid".
82 ///
83 /// Note that only the
84 /// prefix (`foo`) is included in the token, not the separator (which is
85 /// lexed as its own distinct token). In Rust 2021 and later, reserved
86 /// prefixes are reported as errors; in earlier editions, they result in a
87 /// (allowed by default) lint, and are treated as regular identifier
88 /// tokens.
89 UnknownPrefix,
90
91 /// An unknown prefix in a lifetime, like `'foo#`.
92 ///
93 /// Like `UnknownPrefix`, only the `'` and prefix are included in the token
94 /// and not the separator.
95 UnknownPrefixLifetime,
96
97 /// A raw lifetime, e.g. `'r#foo`. In edition < 2021 it will be split into
98 /// several tokens: `'r` and `#` and `foo`.
99 RawLifetime,
100
101 /// Guarded string literal prefix: `#"` or `##`.
102 ///
103 /// Used for reserving "guarded strings" (RFC 3598) in edition 2024.
104 /// Split into the component tokens on older editions.
105 GuardedStrPrefix,
106
107 /// Literals, e.g. `12u8`, `1.0e-40`, `b"123"`. Note that `_` is an invalid
108 /// suffix, but may be present here on string and float literals. Users of
109 /// this type will need to check for and reject that case.
110 ///
111 /// See [LiteralKind] for more details.
112 Literal { kind: LiteralKind, suffix_start: u32 },
113
114 /// A lifetime, e.g. `'a`.
115 Lifetime { starts_with_number: bool },
116
117 /// `;`
118 Semi,
119 /// `,`
120 Comma,
121 /// `.`
122 Dot,
123 /// `(`
124 OpenParen,
125 /// `)`
126 CloseParen,
127 /// `{`
128 OpenBrace,
129 /// `}`
130 CloseBrace,
131 /// `[`
132 OpenBracket,
133 /// `]`
134 CloseBracket,
135 /// `@`
136 At,
137 /// `#`
138 Pound,
139 /// `~`
140 Tilde,
141 /// `?`
142 Question,
143 /// `:`
144 Colon,
145 /// `$`
146 Dollar,
147 /// `=`
148 Eq,
149 /// `!`
150 Bang,
151 /// `<`
152 Lt,
153 /// `>`
154 Gt,
155 /// `-`
156 Minus,
157 /// `&`
158 And,
159 /// `|`
160 Or,
161 /// `+`
162 Plus,
163 /// `*`
164 Star,
165 /// `/`
166 Slash,
167 /// `^`
168 Caret,
169 /// `%`
170 Percent,
171
172 /// Unknown token, not expected by the lexer, e.g. "№"
173 Unknown,
174
175 /// End of input.
176 Eof,
177}
178
179#[derive(Clone, Copy, Debug, PartialEq, Eq)]
180pub enum DocStyle {
181 Outer,
182 Inner,
183}
184
185/// Enum representing the literal types supported by the lexer.
186///
187/// Note that the suffix is *not* considered when deciding the `LiteralKind` in
188/// this type. This means that float literals like `1f32` are classified by this
189/// type as `Int`. (Compare against `rustc_ast::token::LitKind` and
190/// `rustc_ast::ast::LitKind`).
191#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
192pub enum LiteralKind {
193 /// `12_u8`, `0o100`, `0b120i99`, `1f32`.
194 Int { base: Base, empty_int: bool },
195 /// `12.34f32`, `1e3`, but not `1f32`.
196 Float { base: Base, empty_exponent: bool },
197 /// `'a'`, `'\\'`, `'''`, `';`
198 Char { terminated: bool },
199 /// `b'a'`, `b'\\'`, `b'''`, `b';`
200 Byte { terminated: bool },
201 /// `"abc"`, `"abc`
202 Str { terminated: bool },
203 /// `b"abc"`, `b"abc`
204 ByteStr { terminated: bool },
205 /// `c"abc"`, `c"abc`
206 CStr { terminated: bool },
207 /// `r"abc"`, `r#"abc"#`, `r####"ab"###"c"####`, `r#"a`. `None` indicates
208 /// an invalid literal.
209 RawStr { n_hashes: Option<u8> },
210 /// `br"abc"`, `br#"abc"#`, `br####"ab"###"c"####`, `br#"a`. `None`
211 /// indicates an invalid literal.
212 RawByteStr { n_hashes: Option<u8> },
213 /// `cr"abc"`, "cr#"abc"#", `cr#"a`. `None` indicates an invalid literal.
214 RawCStr { n_hashes: Option<u8> },
215}
216
217/// `#"abc"#`, `##"a"` (fewer closing), or even `#"a` (unterminated).
218///
219/// Can capture fewer closing hashes than starting hashes,
220/// for more efficient lexing and better backwards diagnostics.
221#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
222pub struct GuardedStr {
223 pub n_hashes: u32,
224 pub terminated: bool,
225 pub token_len: u32,
226}
227
228#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
229pub enum RawStrError {
230 /// Non `#` characters exist between `r` and `"`, e.g. `r##~"abcde"##`
231 InvalidStarter { bad_char: char },
232 /// The string was not terminated, e.g. `r###"abcde"##`.
233 /// `possible_terminator_offset` is the number of characters after `r` or
234 /// `br` where they may have intended to terminate it.
235 NoTerminator { expected: u32, found: u32, possible_terminator_offset: Option<u32> },
236 /// More than 255 `#`s exist.
237 TooManyDelimiters { found: u32 },
238}
239
240/// Base of numeric literal encoding according to its prefix.
241#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
242pub enum Base {
243 /// Literal starts with "0b".
244 Binary = 2,
245 /// Literal starts with "0o".
246 Octal = 8,
247 /// Literal doesn't contain a prefix.
248 Decimal = 10,
249 /// Literal starts with "0x".
250 Hexadecimal = 16,
251}
252
253/// `rustc` allows files to have a shebang, e.g. "#!/usr/bin/rustrun",
254/// but shebang isn't a part of rust syntax.
255pub fn strip_shebang(input: &str) -> Option<usize> {
256 // Shebang must start with `#!` literally, without any preceding whitespace.
257 // For simplicity we consider any line starting with `#!` a shebang,
258 // regardless of restrictions put on shebangs by specific platforms.
259 if let Some(input_tail) = input.strip_prefix("#!") {
260 // Ok, this is a shebang but if the next non-whitespace token is `[`,
261 // then it may be valid Rust code, so consider it Rust code.
262 let next_non_whitespace_token = tokenize(input_tail).map(|tok| tok.kind).find(|tok| {
263 !matches!(
264 tok,
265 TokenKind::Whitespace
266 | TokenKind::LineComment { doc_style: None }
267 | TokenKind::BlockComment { doc_style: None, .. }
268 )
269 });
270 if next_non_whitespace_token != Some(TokenKind::OpenBracket) {
271 // No other choice than to consider this a shebang.
272 return Some(2 + input_tail.lines().next().unwrap_or_default().len());
273 }
274 }
275 None
276}
277
278/// Validates a raw string literal. Used for getting more information about a
279/// problem with a `RawStr`/`RawByteStr` with a `None` field.
280#[inline]
281pub fn validate_raw_str(input: &str, prefix_len: u32) -> Result<(), RawStrError> {
282 debug_assert!(!input.is_empty());
283 let mut cursor = Cursor::new(input);
284 // Move past the leading `r` or `br`.
285 for _ in 0..prefix_len {
286 cursor.bump().unwrap();
287 }
288 cursor.raw_double_quoted_string(prefix_len).map(|_| ())
289}
290
291/// Creates an iterator that produces tokens from the input string.
292pub fn tokenize(input: &str) -> impl Iterator<Item = Token> {
293 let mut cursor = Cursor::new(input);
294 std::iter::from_fn(move || {
295 let token = cursor.advance_token();
296 if token.kind != TokenKind::Eof { Some(token) } else { None }
297 })
298}
299
300/// True if `c` is considered a whitespace according to Rust language definition.
301/// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
302/// for definitions of these classes.
303pub fn is_whitespace(c: char) -> bool {
304 // This is Pattern_White_Space.
305 //
306 // Note that this set is stable (ie, it doesn't change with different
307 // Unicode versions), so it's ok to just hard-code the values.
308
309 matches!(
310 c,
311 // Usual ASCII suspects
312 '\u{0009}' // \t
313 | '\u{000A}' // \n
314 | '\u{000B}' // vertical tab
315 | '\u{000C}' // form feed
316 | '\u{000D}' // \r
317 | '\u{0020}' // space
318
319 // NEXT LINE from latin1
320 | '\u{0085}'
321
322 // Bidi markers
323 | '\u{200E}' // LEFT-TO-RIGHT MARK
324 | '\u{200F}' // RIGHT-TO-LEFT MARK
325
326 // Dedicated whitespace characters from Unicode
327 | '\u{2028}' // LINE SEPARATOR
328 | '\u{2029}' // PARAGRAPH SEPARATOR
329 )
330}
331
332/// True if `c` is valid as a first character of an identifier.
333/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
334/// a formal definition of valid identifier name.
335pub fn is_id_start(c: char) -> bool {
336 // This is XID_Start OR '_' (which formally is not a XID_Start).
337 c == '_' || unicode_xid::UnicodeXID::is_xid_start(c)
338}
339
340/// True if `c` is valid as a non-first character of an identifier.
341/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
342/// a formal definition of valid identifier name.
343pub fn is_id_continue(c: char) -> bool {
344 unicode_xid::UnicodeXID::is_xid_continue(c)
345}
346
347/// The passed string is lexically an identifier.
348pub fn is_ident(string: &str) -> bool {
349 let mut chars = string.chars();
350 if let Some(start) = chars.next() {
351 is_id_start(start) && chars.all(is_id_continue)
352 } else {
353 false
354 }
355}
356
357impl Cursor<'_> {
358 /// Parses a token from the input string.
359 pub fn advance_token(&mut self) -> Token {
360 let first_char = match self.bump() {
361 Some(c) => c,
362 None => return Token::new(TokenKind::Eof, 0),
363 };
364 let token_kind = match first_char {
365 // Slash, comment or block comment.
366 '/' => match self.first() {
367 '/' => self.line_comment(),
368 '*' => self.block_comment(),
369 _ => Slash,
370 },
371
372 // Whitespace sequence.
373 c if is_whitespace(c) => self.whitespace(),
374
375 // Raw identifier, raw string literal or identifier.
376 'r' => match (self.first(), self.second()) {
377 ('#', c1) if is_id_start(c1) => self.raw_ident(),
378 ('#', _) | ('"', _) => {
379 let res = self.raw_double_quoted_string(1);
380 let suffix_start = self.pos_within_token();
381 if res.is_ok() {
382 self.eat_literal_suffix();
383 }
384 let kind = RawStr { n_hashes: res.ok() };
385 Literal { kind, suffix_start }
386 }
387 _ => self.ident_or_unknown_prefix(),
388 },
389
390 // Byte literal, byte string literal, raw byte string literal or identifier.
391 'b' => self.c_or_byte_string(
392 |terminated| ByteStr { terminated },
393 |n_hashes| RawByteStr { n_hashes },
394 Some(|terminated| Byte { terminated }),
395 ),
396
397 // c-string literal, raw c-string literal or identifier.
398 'c' => self.c_or_byte_string(
399 |terminated| CStr { terminated },
400 |n_hashes| RawCStr { n_hashes },
401 None,
402 ),
403
404 // Identifier (this should be checked after other variant that can
405 // start as identifier).
406 c if is_id_start(c) => self.ident_or_unknown_prefix(),
407
408 // Numeric literal.
409 c @ '0'..='9' => {
410 let literal_kind = self.number(c);
411 let suffix_start = self.pos_within_token();
412 self.eat_literal_suffix();
413 TokenKind::Literal { kind: literal_kind, suffix_start }
414 }
415
416 // Guarded string literal prefix: `#"` or `##`
417 '#' if matches!(self.first(), '"' | '#') => {
418 self.bump();
419 TokenKind::GuardedStrPrefix
420 }
421
422 // One-symbol tokens.
423 ';' => Semi,
424 ',' => Comma,
425 '.' => Dot,
426 '(' => OpenParen,
427 ')' => CloseParen,
428 '{' => OpenBrace,
429 '}' => CloseBrace,
430 '[' => OpenBracket,
431 ']' => CloseBracket,
432 '@' => At,
433 '#' => Pound,
434 '~' => Tilde,
435 '?' => Question,
436 ':' => Colon,
437 '$' => Dollar,
438 '=' => Eq,
439 '!' => Bang,
440 '<' => Lt,
441 '>' => Gt,
442 '-' => Minus,
443 '&' => And,
444 '|' => Or,
445 '+' => Plus,
446 '*' => Star,
447 '^' => Caret,
448 '%' => Percent,
449
450 // Lifetime or character literal.
451 '\'' => self.lifetime_or_char(),
452
453 // String literal.
454 '"' => {
455 let terminated = self.double_quoted_string();
456 let suffix_start = self.pos_within_token();
457 if terminated {
458 self.eat_literal_suffix();
459 }
460 let kind = Str { terminated };
461 Literal { kind, suffix_start }
462 }
463 // Identifier starting with an emoji. Only lexed for graceful error recovery.
464 c if !c.is_ascii() && c.is_emoji_char() => self.invalid_ident(),
465 _ => Unknown,
466 };
467 let res = Token::new(token_kind, self.pos_within_token());
468 self.reset_pos_within_token();
469 res
470 }
471
472 fn line_comment(&mut self) -> TokenKind {
473 debug_assert!(self.prev() == '/' && self.first() == '/');
474 self.bump();
475
476 let doc_style = match self.first() {
477 // `//!` is an inner line doc comment.
478 '!' => Some(DocStyle::Inner),
479 // `////` (more than 3 slashes) is not considered a doc comment.
480 '/' if self.second() != '/' => Some(DocStyle::Outer),
481 _ => None,
482 };
483
484 self.eat_until(b'\n');
485 LineComment { doc_style }
486 }
487
488 fn block_comment(&mut self) -> TokenKind {
489 debug_assert!(self.prev() == '/' && self.first() == '*');
490 self.bump();
491
492 let doc_style = match self.first() {
493 // `/*!` is an inner block doc comment.
494 '!' => Some(DocStyle::Inner),
495 // `/***` (more than 2 stars) is not considered a doc comment.
496 // `/**/` is not considered a doc comment.
497 '*' if !matches!(self.second(), '*' | '/') => Some(DocStyle::Outer),
498 _ => None,
499 };
500
501 let mut depth = 1usize;
502 while let Some(c) = self.bump() {
503 match c {
504 '/' if self.first() == '*' => {
505 self.bump();
506 depth += 1;
507 }
508 '*' if self.first() == '/' => {
509 self.bump();
510 depth -= 1;
511 if depth == 0 {
512 // This block comment is closed, so for a construction like "/* */ */"
513 // there will be a successfully parsed block comment "/* */"
514 // and " */" will be processed separately.
515 break;
516 }
517 }
518 _ => (),
519 }
520 }
521
522 BlockComment { doc_style, terminated: depth == 0 }
523 }
524
525 fn whitespace(&mut self) -> TokenKind {
526 debug_assert!(is_whitespace(self.prev()));
527 self.eat_while(is_whitespace);
528 Whitespace
529 }
530
531 fn raw_ident(&mut self) -> TokenKind {
532 debug_assert!(self.prev() == 'r' && self.first() == '#' && is_id_start(self.second()));
533 // Eat "#" symbol.
534 self.bump();
535 // Eat the identifier part of RawIdent.
536 self.eat_identifier();
537 RawIdent
538 }
539
540 fn ident_or_unknown_prefix(&mut self) -> TokenKind {
541 debug_assert!(is_id_start(self.prev()));
542 // Start is already eaten, eat the rest of identifier.
543 self.eat_while(is_id_continue);
544 // Known prefixes must have been handled earlier. So if
545 // we see a prefix here, it is definitely an unknown prefix.
546 match self.first() {
547 '#' | '"' | '\'' => UnknownPrefix,
548 c if !c.is_ascii() && c.is_emoji_char() => self.invalid_ident(),
549 _ => Ident,
550 }
551 }
552
553 fn invalid_ident(&mut self) -> TokenKind {
554 // Start is already eaten, eat the rest of identifier.
555 self.eat_while(|c| {
556 const ZERO_WIDTH_JOINER: char = '\u{200d}';
557 is_id_continue(c) || (!c.is_ascii() && c.is_emoji_char()) || c == ZERO_WIDTH_JOINER
558 });
559 // An invalid identifier followed by '#' or '"' or '\'' could be
560 // interpreted as an invalid literal prefix. We don't bother doing that
561 // because the treatment of invalid identifiers and invalid prefixes
562 // would be the same.
563 InvalidIdent
564 }
565
566 fn c_or_byte_string(
567 &mut self,
568 mk_kind: fn(bool) -> LiteralKind,
569 mk_kind_raw: fn(Option<u8>) -> LiteralKind,
570 single_quoted: Option<fn(bool) -> LiteralKind>,
571 ) -> TokenKind {
572 match (self.first(), self.second(), single_quoted) {
573 ('\'', _, Some(single_quoted)) => {
574 self.bump();
575 let terminated = self.single_quoted_string();
576 let suffix_start = self.pos_within_token();
577 if terminated {
578 self.eat_literal_suffix();
579 }
580 let kind = single_quoted(terminated);
581 Literal { kind, suffix_start }
582 }
583 ('"', _, _) => {
584 self.bump();
585 let terminated = self.double_quoted_string();
586 let suffix_start = self.pos_within_token();
587 if terminated {
588 self.eat_literal_suffix();
589 }
590 let kind = mk_kind(terminated);
591 Literal { kind, suffix_start }
592 }
593 ('r', '"', _) | ('r', '#', _) => {
594 self.bump();
595 let res = self.raw_double_quoted_string(2);
596 let suffix_start = self.pos_within_token();
597 if res.is_ok() {
598 self.eat_literal_suffix();
599 }
600 let kind = mk_kind_raw(res.ok());
601 Literal { kind, suffix_start }
602 }
603 _ => self.ident_or_unknown_prefix(),
604 }
605 }
606
607 fn number(&mut self, first_digit: char) -> LiteralKind {
608 debug_assert!('0' <= self.prev() && self.prev() <= '9');
609 let mut base = Base::Decimal;
610 if first_digit == '0' {
611 // Attempt to parse encoding base.
612 match self.first() {
613 'b' => {
614 base = Base::Binary;
615 self.bump();
616 if !self.eat_decimal_digits() {
617 return Int { base, empty_int: true };
618 }
619 }
620 'o' => {
621 base = Base::Octal;
622 self.bump();
623 if !self.eat_decimal_digits() {
624 return Int { base, empty_int: true };
625 }
626 }
627 'x' => {
628 base = Base::Hexadecimal;
629 self.bump();
630 if !self.eat_hexadecimal_digits() {
631 return Int { base, empty_int: true };
632 }
633 }
634 // Not a base prefix; consume additional digits.
635 '0'..='9' | '_' => {
636 self.eat_decimal_digits();
637 }
638
639 // Also not a base prefix; nothing more to do here.
640 '.' | 'e' | 'E' => {}
641
642 // Just a 0.
643 _ => return Int { base, empty_int: false },
644 }
645 } else {
646 // No base prefix, parse number in the usual way.
647 self.eat_decimal_digits();
648 };
649
650 match self.first() {
651 // Don't be greedy if this is actually an
652 // integer literal followed by field/method access or a range pattern
653 // (`0..2` and `12.foo()`)
654 '.' if self.second() != '.' && !is_id_start(self.second()) => {
655 // might have stuff after the ., and if it does, it needs to start
656 // with a number
657 self.bump();
658 let mut empty_exponent = false;
659 if self.first().is_ascii_digit() {
660 self.eat_decimal_digits();
661 match self.first() {
662 'e' | 'E' => {
663 self.bump();
664 empty_exponent = !self.eat_float_exponent();
665 }
666 _ => (),
667 }
668 }
669 Float { base, empty_exponent }
670 }
671 'e' | 'E' => {
672 self.bump();
673 let empty_exponent = !self.eat_float_exponent();
674 Float { base, empty_exponent }
675 }
676 _ => Int { base, empty_int: false },
677 }
678 }
679
680 fn lifetime_or_char(&mut self) -> TokenKind {
681 debug_assert!(self.prev() == '\'');
682
683 let can_be_a_lifetime = if self.second() == '\'' {
684 // It's surely not a lifetime.
685 false
686 } else {
687 // If the first symbol is valid for identifier, it can be a lifetime.
688 // Also check if it's a number for a better error reporting (so '0 will
689 // be reported as invalid lifetime and not as unterminated char literal).
690 is_id_start(self.first()) || self.first().is_ascii_digit()
691 };
692
693 if !can_be_a_lifetime {
694 let terminated = self.single_quoted_string();
695 let suffix_start = self.pos_within_token();
696 if terminated {
697 self.eat_literal_suffix();
698 }
699 let kind = Char { terminated };
700 return Literal { kind, suffix_start };
701 }
702
703 if self.first() == 'r' && self.second() == '#' && is_id_start(self.third()) {
704 // Eat "r" and `#`, and identifier start characters.
705 self.bump();
706 self.bump();
707 self.bump();
708 self.eat_while(is_id_continue);
709 return RawLifetime;
710 }
711
712 // Either a lifetime or a character literal with
713 // length greater than 1.
714 let starts_with_number = self.first().is_ascii_digit();
715
716 // Skip the literal contents.
717 // First symbol can be a number (which isn't a valid identifier start),
718 // so skip it without any checks.
719 self.bump();
720 self.eat_while(is_id_continue);
721
722 match self.first() {
723 // Check if after skipping literal contents we've met a closing
724 // single quote (which means that user attempted to create a
725 // string with single quotes).
726 '\'' => {
727 self.bump();
728 let kind = Char { terminated: true };
729 Literal { kind, suffix_start: self.pos_within_token() }
730 }
731 '#' if !starts_with_number => UnknownPrefixLifetime,
732 _ => Lifetime { starts_with_number },
733 }
734 }
735
736 fn single_quoted_string(&mut self) -> bool {
737 debug_assert!(self.prev() == '\'');
738 // Check if it's a one-symbol literal.
739 if self.second() == '\'' && self.first() != '\\' {
740 self.bump();
741 self.bump();
742 return true;
743 }
744
745 // Literal has more than one symbol.
746
747 // Parse until either quotes are terminated or error is detected.
748 loop {
749 match self.first() {
750 // Quotes are terminated, finish parsing.
751 '\'' => {
752 self.bump();
753 return true;
754 }
755 // Probably beginning of the comment, which we don't want to include
756 // to the error report.
757 '/' => break,
758 // Newline without following '\'' means unclosed quote, stop parsing.
759 '\n' if self.second() != '\'' => break,
760 // End of file, stop parsing.
761 EOF_CHAR if self.is_eof() => break,
762 // Escaped slash is considered one character, so bump twice.
763 '\\' => {
764 self.bump();
765 self.bump();
766 }
767 // Skip the character.
768 _ => {
769 self.bump();
770 }
771 }
772 }
773 // String was not terminated.
774 false
775 }
776
777 /// Eats double-quoted string and returns true
778 /// if string is terminated.
779 fn double_quoted_string(&mut self) -> bool {
780 debug_assert!(self.prev() == '"');
781 while let Some(c) = self.bump() {
782 match c {
783 '"' => {
784 return true;
785 }
786 '\\' if self.first() == '\\' || self.first() == '"' => {
787 // Bump again to skip escaped character.
788 self.bump();
789 }
790 _ => (),
791 }
792 }
793 // End of file reached.
794 false
795 }
796
797 /// Attempt to lex for a guarded string literal.
798 ///
799 /// Used by `rustc_parse::lexer` to lex for guarded strings
800 /// conditionally based on edition.
801 ///
802 /// Note: this will not reset the `Cursor` when a
803 /// guarded string is not found. It is the caller's
804 /// responsibility to do so.
805 pub fn guarded_double_quoted_string(&mut self) -> Option<GuardedStr> {
806 debug_assert!(self.prev() != '#');
807
808 let mut n_start_hashes: u32 = 0;
809 while self.first() == '#' {
810 n_start_hashes += 1;
811 self.bump();
812 }
813
814 if self.first() != '"' {
815 return None;
816 }
817 self.bump();
818 debug_assert!(self.prev() == '"');
819
820 // Lex the string itself as a normal string literal
821 // so we can recover that for older editions later.
822 let terminated = self.double_quoted_string();
823 if !terminated {
824 let token_len = self.pos_within_token();
825 self.reset_pos_within_token();
826
827 return Some(GuardedStr { n_hashes: n_start_hashes, terminated: false, token_len });
828 }
829
830 // Consume closing '#' symbols.
831 // Note that this will not consume extra trailing `#` characters:
832 // `###"abcde"####` is lexed as a `GuardedStr { n_end_hashes: 3, .. }`
833 // followed by a `#` token.
834 let mut n_end_hashes = 0;
835 while self.first() == '#' && n_end_hashes < n_start_hashes {
836 n_end_hashes += 1;
837 self.bump();
838 }
839
840 // Reserved syntax, always an error, so it doesn't matter if
841 // `n_start_hashes != n_end_hashes`.
842
843 self.eat_literal_suffix();
844
845 let token_len = self.pos_within_token();
846 self.reset_pos_within_token();
847
848 Some(GuardedStr { n_hashes: n_start_hashes, terminated: true, token_len })
849 }
850
851 /// Eats the double-quoted string and returns `n_hashes` and an error if encountered.
852 fn raw_double_quoted_string(&mut self, prefix_len: u32) -> Result<u8, RawStrError> {
853 // Wrap the actual function to handle the error with too many hashes.
854 // This way, it eats the whole raw string.
855 let n_hashes = self.raw_string_unvalidated(prefix_len)?;
856 // Only up to 255 `#`s are allowed in raw strings
857 match u8::try_from(n_hashes) {
858 Ok(num) => Ok(num),
859 Err(_) => Err(RawStrError::TooManyDelimiters { found: n_hashes }),
860 }
861 }
862
863 fn raw_string_unvalidated(&mut self, prefix_len: u32) -> Result<u32, RawStrError> {
864 debug_assert!(self.prev() == 'r');
865 let start_pos = self.pos_within_token();
866 let mut possible_terminator_offset = None;
867 let mut max_hashes = 0;
868
869 // Count opening '#' symbols.
870 let mut eaten = 0;
871 while self.first() == '#' {
872 eaten += 1;
873 self.bump();
874 }
875 let n_start_hashes = eaten;
876
877 // Check that string is started.
878 match self.bump() {
879 Some('"') => (),
880 c => {
881 let c = c.unwrap_or(EOF_CHAR);
882 return Err(RawStrError::InvalidStarter { bad_char: c });
883 }
884 }
885
886 // Skip the string contents and on each '#' character met, check if this is
887 // a raw string termination.
888 loop {
889 self.eat_until(b'"');
890
891 if self.is_eof() {
892 return Err(RawStrError::NoTerminator {
893 expected: n_start_hashes,
894 found: max_hashes,
895 possible_terminator_offset,
896 });
897 }
898
899 // Eat closing double quote.
900 self.bump();
901
902 // Check that amount of closing '#' symbols
903 // is equal to the amount of opening ones.
904 // Note that this will not consume extra trailing `#` characters:
905 // `r###"abcde"####` is lexed as a `RawStr { n_hashes: 3 }`
906 // followed by a `#` token.
907 let mut n_end_hashes = 0;
908 while self.first() == '#' && n_end_hashes < n_start_hashes {
909 n_end_hashes += 1;
910 self.bump();
911 }
912
913 if n_end_hashes == n_start_hashes {
914 return Ok(n_start_hashes);
915 } else if n_end_hashes > max_hashes {
916 // Keep track of possible terminators to give a hint about
917 // where there might be a missing terminator
918 possible_terminator_offset =
919 Some(self.pos_within_token() - start_pos - n_end_hashes + prefix_len);
920 max_hashes = n_end_hashes;
921 }
922 }
923 }
924
925 fn eat_decimal_digits(&mut self) -> bool {
926 let mut has_digits = false;
927 loop {
928 match self.first() {
929 '_' => {
930 self.bump();
931 }
932 '0'..='9' => {
933 has_digits = true;
934 self.bump();
935 }
936 _ => break,
937 }
938 }
939 has_digits
940 }
941
942 fn eat_hexadecimal_digits(&mut self) -> bool {
943 let mut has_digits = false;
944 loop {
945 match self.first() {
946 '_' => {
947 self.bump();
948 }
949 '0'..='9' | 'a'..='f' | 'A'..='F' => {
950 has_digits = true;
951 self.bump();
952 }
953 _ => break,
954 }
955 }
956 has_digits
957 }
958
959 /// Eats the float exponent. Returns true if at least one digit was met,
960 /// and returns false otherwise.
961 fn eat_float_exponent(&mut self) -> bool {
962 debug_assert!(self.prev() == 'e' || self.prev() == 'E');
963 if self.first() == '-' || self.first() == '+' {
964 self.bump();
965 }
966 self.eat_decimal_digits()
967 }
968
969 // Eats the suffix of the literal, e.g. "u8".
970 fn eat_literal_suffix(&mut self) {
971 self.eat_identifier();
972 }
973
974 // Eats the identifier. Note: succeeds on `_`, which isn't a valid
975 // identifier.
976 fn eat_identifier(&mut self) {
977 if !is_id_start(self.first()) {
978 return;
979 }
980 self.bump();
981
982 self.eat_while(is_id_continue);
983 }
984}