rustc_lexer/lib.rs
1//! Low-level Rust lexer.
2//!
3//! The idea with `rustc_lexer` is to make a reusable library,
4//! by separating out pure lexing and rustc-specific concerns, like spans,
5//! error reporting, and interning. So, rustc_lexer operates directly on `&str`,
6//! produces simple tokens which are a pair of type-tag and a bit of original text,
7//! and does not report errors, instead storing them as flags on the token.
8//!
9//! Tokens produced by this lexer are not yet ready for parsing the Rust syntax.
10//! For that see [`rustc_parse::lexer`], which converts this basic token stream
11//! into wide tokens used by actual parser.
12//!
13//! The purpose of this crate is to convert raw sources into a labeled sequence
14//! of well-known token types, so building an actual Rust token stream will
15//! be easier.
16//!
17//! The main entity of this crate is the [`TokenKind`] enum which represents common
18//! lexeme types.
19//!
20//! [`rustc_parse::lexer`]: ../rustc_parse/lexer/index.html
21
22// tidy-alphabetical-start
23// We want to be able to build this crate with a stable compiler,
24// so no `#![feature]` attributes should be added.
25#![deny(unstable_features)]
26// tidy-alphabetical-end
27
28mod cursor;
29
30#[cfg(test)]
31mod tests;
32
33use LiteralKind::*;
34use TokenKind::*;
35use cursor::EOF_CHAR;
36pub use cursor::{Cursor, FrontmatterAllowed};
37use unicode_properties::UnicodeEmoji;
38pub use unicode_xid::UNICODE_VERSION as UNICODE_XID_VERSION;
39
40/// Parsed token.
41/// It doesn't contain information about data that has been parsed,
42/// only the type of the token and its size.
43#[derive(Debug)]
44pub struct Token {
45 pub kind: TokenKind,
46 pub len: u32,
47}
48
49impl Token {
50 fn new(kind: TokenKind, len: u32) -> Token {
51 Token { kind, len }
52 }
53}
54
55/// Enum representing common lexeme types.
56#[derive(Clone, Copy, Debug, PartialEq, Eq)]
57pub enum TokenKind {
58 /// A line comment, e.g. `// comment`.
59 LineComment {
60 doc_style: Option<DocStyle>,
61 },
62
63 /// A block comment, e.g. `/* block comment */`.
64 ///
65 /// Block comments can be recursive, so a sequence like `/* /* */`
66 /// will not be considered terminated and will result in a parsing error.
67 BlockComment {
68 doc_style: Option<DocStyle>,
69 terminated: bool,
70 },
71
72 /// Any whitespace character sequence.
73 Whitespace,
74
75 Frontmatter {
76 has_invalid_preceding_whitespace: bool,
77 invalid_infostring: bool,
78 },
79
80 /// An identifier or keyword, e.g. `ident` or `continue`.
81 Ident,
82
83 /// An identifier that is invalid because it contains emoji.
84 InvalidIdent,
85
86 /// A raw identifier, e.g. "r#ident".
87 RawIdent,
88
89 /// An unknown literal prefix, like `foo#`, `foo'`, `foo"`. Excludes
90 /// literal prefixes that contain emoji, which are considered "invalid".
91 ///
92 /// Note that only the
93 /// prefix (`foo`) is included in the token, not the separator (which is
94 /// lexed as its own distinct token). In Rust 2021 and later, reserved
95 /// prefixes are reported as errors; in earlier editions, they result in a
96 /// (allowed by default) lint, and are treated as regular identifier
97 /// tokens.
98 UnknownPrefix,
99
100 /// An unknown prefix in a lifetime, like `'foo#`.
101 ///
102 /// Like `UnknownPrefix`, only the `'` and prefix are included in the token
103 /// and not the separator.
104 UnknownPrefixLifetime,
105
106 /// A raw lifetime, e.g. `'r#foo`. In edition < 2021 it will be split into
107 /// several tokens: `'r` and `#` and `foo`.
108 RawLifetime,
109
110 /// Guarded string literal prefix: `#"` or `##`.
111 ///
112 /// Used for reserving "guarded strings" (RFC 3598) in edition 2024.
113 /// Split into the component tokens on older editions.
114 GuardedStrPrefix,
115
116 /// Literals, e.g. `12u8`, `1.0e-40`, `b"123"`. Note that `_` is an invalid
117 /// suffix, but may be present here on string and float literals. Users of
118 /// this type will need to check for and reject that case.
119 ///
120 /// See [LiteralKind] for more details.
121 Literal {
122 kind: LiteralKind,
123 suffix_start: u32,
124 },
125
126 /// A lifetime, e.g. `'a`.
127 Lifetime {
128 starts_with_number: bool,
129 },
130
131 /// `;`
132 Semi,
133 /// `,`
134 Comma,
135 /// `.`
136 Dot,
137 /// `(`
138 OpenParen,
139 /// `)`
140 CloseParen,
141 /// `{`
142 OpenBrace,
143 /// `}`
144 CloseBrace,
145 /// `[`
146 OpenBracket,
147 /// `]`
148 CloseBracket,
149 /// `@`
150 At,
151 /// `#`
152 Pound,
153 /// `~`
154 Tilde,
155 /// `?`
156 Question,
157 /// `:`
158 Colon,
159 /// `$`
160 Dollar,
161 /// `=`
162 Eq,
163 /// `!`
164 Bang,
165 /// `<`
166 Lt,
167 /// `>`
168 Gt,
169 /// `-`
170 Minus,
171 /// `&`
172 And,
173 /// `|`
174 Or,
175 /// `+`
176 Plus,
177 /// `*`
178 Star,
179 /// `/`
180 Slash,
181 /// `^`
182 Caret,
183 /// `%`
184 Percent,
185
186 /// Unknown token, not expected by the lexer, e.g. "№"
187 Unknown,
188
189 /// End of input.
190 Eof,
191}
192
193#[derive(Clone, Copy, Debug, PartialEq, Eq)]
194pub enum DocStyle {
195 Outer,
196 Inner,
197}
198
199/// Enum representing the literal types supported by the lexer.
200///
201/// Note that the suffix is *not* considered when deciding the `LiteralKind` in
202/// this type. This means that float literals like `1f32` are classified by this
203/// type as `Int`. (Compare against `rustc_ast::token::LitKind` and
204/// `rustc_ast::ast::LitKind`).
205#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
206pub enum LiteralKind {
207 /// `12_u8`, `0o100`, `0b120i99`, `1f32`.
208 Int { base: Base, empty_int: bool },
209 /// `12.34f32`, `1e3`, but not `1f32`.
210 Float { base: Base, empty_exponent: bool },
211 /// `'a'`, `'\\'`, `'''`, `';`
212 Char { terminated: bool },
213 /// `b'a'`, `b'\\'`, `b'''`, `b';`
214 Byte { terminated: bool },
215 /// `"abc"`, `"abc`
216 Str { terminated: bool },
217 /// `b"abc"`, `b"abc`
218 ByteStr { terminated: bool },
219 /// `c"abc"`, `c"abc`
220 CStr { terminated: bool },
221 /// `r"abc"`, `r#"abc"#`, `r####"ab"###"c"####`, `r#"a`. `None` indicates
222 /// an invalid literal.
223 RawStr { n_hashes: Option<u8> },
224 /// `br"abc"`, `br#"abc"#`, `br####"ab"###"c"####`, `br#"a`. `None`
225 /// indicates an invalid literal.
226 RawByteStr { n_hashes: Option<u8> },
227 /// `cr"abc"`, "cr#"abc"#", `cr#"a`. `None` indicates an invalid literal.
228 RawCStr { n_hashes: Option<u8> },
229}
230
231/// `#"abc"#`, `##"a"` (fewer closing), or even `#"a` (unterminated).
232///
233/// Can capture fewer closing hashes than starting hashes,
234/// for more efficient lexing and better backwards diagnostics.
235#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
236pub struct GuardedStr {
237 pub n_hashes: u32,
238 pub terminated: bool,
239 pub token_len: u32,
240}
241
242#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
243pub enum RawStrError {
244 /// Non `#` characters exist between `r` and `"`, e.g. `r##~"abcde"##`
245 InvalidStarter { bad_char: char },
246 /// The string was not terminated, e.g. `r###"abcde"##`.
247 /// `possible_terminator_offset` is the number of characters after `r` or
248 /// `br` where they may have intended to terminate it.
249 NoTerminator { expected: u32, found: u32, possible_terminator_offset: Option<u32> },
250 /// More than 255 `#`s exist.
251 TooManyDelimiters { found: u32 },
252}
253
254/// Base of numeric literal encoding according to its prefix.
255#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)]
256pub enum Base {
257 /// Literal starts with "0b".
258 Binary = 2,
259 /// Literal starts with "0o".
260 Octal = 8,
261 /// Literal doesn't contain a prefix.
262 Decimal = 10,
263 /// Literal starts with "0x".
264 Hexadecimal = 16,
265}
266
267/// `rustc` allows files to have a shebang, e.g. "#!/usr/bin/rustrun",
268/// but shebang isn't a part of rust syntax.
269pub fn strip_shebang(input: &str) -> Option<usize> {
270 // Shebang must start with `#!` literally, without any preceding whitespace.
271 // For simplicity we consider any line starting with `#!` a shebang,
272 // regardless of restrictions put on shebangs by specific platforms.
273 if let Some(input_tail) = input.strip_prefix("#!") {
274 // Ok, this is a shebang but if the next non-whitespace token is `[`,
275 // then it may be valid Rust code, so consider it Rust code.
276 let next_non_whitespace_token =
277 tokenize(input_tail, FrontmatterAllowed::No).map(|tok| tok.kind).find(|tok| {
278 !matches!(
279 tok,
280 TokenKind::Whitespace
281 | TokenKind::LineComment { doc_style: None }
282 | TokenKind::BlockComment { doc_style: None, .. }
283 )
284 });
285 if next_non_whitespace_token != Some(TokenKind::OpenBracket) {
286 // No other choice than to consider this a shebang.
287 return Some(2 + input_tail.lines().next().unwrap_or_default().len());
288 }
289 }
290 None
291}
292
293/// Validates a raw string literal. Used for getting more information about a
294/// problem with a `RawStr`/`RawByteStr` with a `None` field.
295#[inline]
296pub fn validate_raw_str(input: &str, prefix_len: u32) -> Result<(), RawStrError> {
297 debug_assert!(!input.is_empty());
298 let mut cursor = Cursor::new(input, FrontmatterAllowed::No);
299 // Move past the leading `r` or `br`.
300 for _ in 0..prefix_len {
301 cursor.bump().unwrap();
302 }
303 cursor.raw_double_quoted_string(prefix_len).map(|_| ())
304}
305
306/// Creates an iterator that produces tokens from the input string.
307///
308/// When parsing a full Rust document,
309/// first [`strip_shebang`] and then allow frontmatters with [`FrontmatterAllowed::Yes`].
310///
311/// When tokenizing a slice of a document, be sure to disallow frontmatters with [`FrontmatterAllowed::No`]
312pub fn tokenize(
313 input: &str,
314 frontmatter_allowed: FrontmatterAllowed,
315) -> impl Iterator<Item = Token> {
316 let mut cursor = Cursor::new(input, frontmatter_allowed);
317 std::iter::from_fn(move || {
318 let token = cursor.advance_token();
319 if token.kind != TokenKind::Eof { Some(token) } else { None }
320 })
321}
322
323/// True if `c` is considered a whitespace according to Rust language definition.
324/// See [Rust language reference](https://doc.rust-lang.org/reference/whitespace.html)
325/// for definitions of these classes.
326pub fn is_whitespace(c: char) -> bool {
327 // This is Pattern_White_Space.
328 //
329 // Note that this set is stable (ie, it doesn't change with different
330 // Unicode versions), so it's ok to just hard-code the values.
331
332 matches!(
333 c,
334 // End-of-line characters
335 | '\u{000A}' // line feed (\n)
336 | '\u{000B}' // vertical tab
337 | '\u{000C}' // form feed
338 | '\u{000D}' // carriage return (\r)
339 | '\u{0085}' // next line (from latin1)
340 | '\u{2028}' // LINE SEPARATOR
341 | '\u{2029}' // PARAGRAPH SEPARATOR
342
343 // `Default_Ignorable_Code_Point` characters
344 | '\u{200E}' // LEFT-TO-RIGHT MARK
345 | '\u{200F}' // RIGHT-TO-LEFT MARK
346
347 // Horizontal space characters
348 | '\u{0009}' // tab (\t)
349 | '\u{0020}' // space
350 )
351}
352
353/// True if `c` is considered horizontal whitespace according to Rust language definition.
354pub fn is_horizontal_whitespace(c: char) -> bool {
355 // This is Pattern_White_Space.
356 //
357 // Note that this set is stable (ie, it doesn't change with different
358 // Unicode versions), so it's ok to just hard-code the values.
359
360 matches!(
361 c,
362 // Horizontal space characters
363 '\u{0009}' // tab (\t)
364 | '\u{0020}' // space
365 )
366}
367
368/// True if `c` is valid as a first character of an identifier.
369/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
370/// a formal definition of valid identifier name.
371pub fn is_id_start(c: char) -> bool {
372 // This is XID_Start OR '_' (which formally is not a XID_Start).
373 c == '_' || unicode_xid::UnicodeXID::is_xid_start(c)
374}
375
376/// True if `c` is valid as a non-first character of an identifier.
377/// See [Rust language reference](https://doc.rust-lang.org/reference/identifiers.html) for
378/// a formal definition of valid identifier name.
379pub fn is_id_continue(c: char) -> bool {
380 unicode_xid::UnicodeXID::is_xid_continue(c)
381}
382
383/// The passed string is lexically an identifier.
384pub fn is_ident(string: &str) -> bool {
385 let mut chars = string.chars();
386 if let Some(start) = chars.next() {
387 is_id_start(start) && chars.all(is_id_continue)
388 } else {
389 false
390 }
391}
392
393impl Cursor<'_> {
394 /// Parses a token from the input string.
395 pub fn advance_token(&mut self) -> Token {
396 let Some(first_char) = self.bump() else {
397 return Token::new(TokenKind::Eof, 0);
398 };
399
400 let token_kind = match first_char {
401 c if matches!(self.frontmatter_allowed, FrontmatterAllowed::Yes)
402 && is_whitespace(c) =>
403 {
404 let mut last = first_char;
405 while is_whitespace(self.first()) {
406 let Some(c) = self.bump() else {
407 break;
408 };
409 last = c;
410 }
411 // invalid frontmatter opening as whitespace preceding it isn't newline.
412 // combine the whitespace and the frontmatter to a single token as we shall
413 // error later.
414 if last != '\n' && self.as_str().starts_with("---") {
415 self.bump();
416 self.frontmatter(true)
417 } else {
418 Whitespace
419 }
420 }
421 '-' if matches!(self.frontmatter_allowed, FrontmatterAllowed::Yes)
422 && self.as_str().starts_with("--") =>
423 {
424 // happy path
425 self.frontmatter(false)
426 }
427 // Slash, comment or block comment.
428 '/' => match self.first() {
429 '/' => self.line_comment(),
430 '*' => self.block_comment(),
431 _ => Slash,
432 },
433
434 // Whitespace sequence.
435 c if is_whitespace(c) => self.whitespace(),
436
437 // Raw identifier, raw string literal or identifier.
438 'r' => match (self.first(), self.second()) {
439 ('#', c1) if is_id_start(c1) => self.raw_ident(),
440 ('#', _) | ('"', _) => {
441 let res = self.raw_double_quoted_string(1);
442 let suffix_start = self.pos_within_token();
443 if res.is_ok() {
444 self.eat_literal_suffix();
445 }
446 let kind = RawStr { n_hashes: res.ok() };
447 Literal { kind, suffix_start }
448 }
449 _ => self.ident_or_unknown_prefix(),
450 },
451
452 // Byte literal, byte string literal, raw byte string literal or identifier.
453 'b' => self.c_or_byte_string(
454 |terminated| ByteStr { terminated },
455 |n_hashes| RawByteStr { n_hashes },
456 Some(|terminated| Byte { terminated }),
457 ),
458
459 // c-string literal, raw c-string literal or identifier.
460 'c' => self.c_or_byte_string(
461 |terminated| CStr { terminated },
462 |n_hashes| RawCStr { n_hashes },
463 None,
464 ),
465
466 // Identifier (this should be checked after other variant that can
467 // start as identifier).
468 c if is_id_start(c) => self.ident_or_unknown_prefix(),
469
470 // Numeric literal.
471 c @ '0'..='9' => {
472 let literal_kind = self.number(c);
473 let suffix_start = self.pos_within_token();
474 self.eat_literal_suffix();
475 TokenKind::Literal { kind: literal_kind, suffix_start }
476 }
477
478 // Guarded string literal prefix: `#"` or `##`
479 '#' if matches!(self.first(), '"' | '#') => {
480 self.bump();
481 TokenKind::GuardedStrPrefix
482 }
483
484 // One-symbol tokens.
485 ';' => Semi,
486 ',' => Comma,
487 '.' => Dot,
488 '(' => OpenParen,
489 ')' => CloseParen,
490 '{' => OpenBrace,
491 '}' => CloseBrace,
492 '[' => OpenBracket,
493 ']' => CloseBracket,
494 '@' => At,
495 '#' => Pound,
496 '~' => Tilde,
497 '?' => Question,
498 ':' => Colon,
499 '$' => Dollar,
500 '=' => Eq,
501 '!' => Bang,
502 '<' => Lt,
503 '>' => Gt,
504 '-' => Minus,
505 '&' => And,
506 '|' => Or,
507 '+' => Plus,
508 '*' => Star,
509 '^' => Caret,
510 '%' => Percent,
511
512 // Lifetime or character literal.
513 '\'' => self.lifetime_or_char(),
514
515 // String literal.
516 '"' => {
517 let terminated = self.double_quoted_string();
518 let suffix_start = self.pos_within_token();
519 if terminated {
520 self.eat_literal_suffix();
521 }
522 let kind = Str { terminated };
523 Literal { kind, suffix_start }
524 }
525 // Identifier starting with an emoji. Only lexed for graceful error recovery.
526 c if !c.is_ascii() && c.is_emoji_char() => self.invalid_ident(),
527 _ => Unknown,
528 };
529 if matches!(self.frontmatter_allowed, FrontmatterAllowed::Yes)
530 && !matches!(token_kind, Whitespace)
531 {
532 // stop allowing frontmatters after first non-whitespace token
533 self.frontmatter_allowed = FrontmatterAllowed::No;
534 }
535 let res = Token::new(token_kind, self.pos_within_token());
536 self.reset_pos_within_token();
537 res
538 }
539
540 /// Given that one `-` was eaten, eat the rest of the frontmatter.
541 fn frontmatter(&mut self, has_invalid_preceding_whitespace: bool) -> TokenKind {
542 debug_assert_eq!('-', self.prev());
543
544 let pos = self.pos_within_token();
545 self.eat_while(|c| c == '-');
546
547 // one `-` is eaten by the caller.
548 let length_opening = self.pos_within_token() - pos + 1;
549
550 // must be ensured by the caller
551 debug_assert!(length_opening >= 3);
552
553 // whitespace between the opening and the infostring.
554 self.eat_while(|ch| ch != '\n' && is_horizontal_whitespace(ch));
555
556 // copied from `eat_identifier`, but allows `-` and `.` in infostring to allow something like
557 // `---Cargo.toml` as a valid opener
558 if is_id_start(self.first()) {
559 self.bump();
560 self.eat_while(|c| is_id_continue(c) || c == '-' || c == '.');
561 }
562
563 self.eat_while(|ch| ch != '\n' && is_horizontal_whitespace(ch));
564 let invalid_infostring = self.first() != '\n';
565
566 let mut found = false;
567 let nl_fence_pattern = format!("\n{:-<1$}", "", length_opening as usize);
568 if let Some(closing) = self.as_str().find(&nl_fence_pattern) {
569 // candidate found
570 self.bump_bytes(closing + nl_fence_pattern.len());
571 // in case like
572 // ---cargo
573 // --- blahblah
574 // or
575 // ---cargo
576 // ----
577 // combine those stuff into this frontmatter token such that it gets detected later.
578 self.eat_until(b'\n');
579 found = true;
580 }
581
582 if !found {
583 // recovery strategy: a closing statement might have preceding whitespace/newline
584 // but not have enough dashes to properly close. In this case, we eat until there,
585 // and report a mismatch in the parser.
586 let mut rest = self.as_str();
587 // We can look for a shorter closing (starting with four dashes but closing with three)
588 // and other indications that Rust has started and the infostring has ended.
589 let mut potential_closing = rest
590 .find("\n---")
591 // n.b. only in the case where there are dashes, we move the index to the line where
592 // the dashes start as we eat to include that line. For other cases those are Rust code
593 // and not included in the frontmatter.
594 .map(|x| x + 1)
595 .or_else(|| rest.find("\nuse "))
596 .or_else(|| rest.find("\n//!"))
597 .or_else(|| rest.find("\n#!["));
598
599 if potential_closing.is_none() {
600 // a less fortunate recovery if all else fails which finds any dashes preceded by whitespace
601 // on a standalone line. Might be wrong.
602 let mut base_index = 0;
603 while let Some(closing) = rest.find("---") {
604 let preceding_chars_start = rest[..closing].rfind("\n").map_or(0, |i| i + 1);
605 if rest[preceding_chars_start..closing].chars().all(is_horizontal_whitespace) {
606 // candidate found
607 potential_closing = Some(closing + base_index);
608 break;
609 } else {
610 rest = &rest[closing + 3..];
611 base_index += closing + 3;
612 }
613 }
614 }
615
616 if let Some(potential_closing) = potential_closing {
617 // bump to the potential closing, and eat everything on that line.
618 self.bump_bytes(potential_closing);
619 self.eat_until(b'\n');
620 } else {
621 // eat everything. this will get reported as an unclosed frontmatter.
622 self.eat_while(|_| true);
623 }
624 }
625
626 Frontmatter { has_invalid_preceding_whitespace, invalid_infostring }
627 }
628
629 fn line_comment(&mut self) -> TokenKind {
630 debug_assert!(self.prev() == '/' && self.first() == '/');
631 self.bump();
632
633 let doc_style = match self.first() {
634 // `//!` is an inner line doc comment.
635 '!' => Some(DocStyle::Inner),
636 // `////` (more than 3 slashes) is not considered a doc comment.
637 '/' if self.second() != '/' => Some(DocStyle::Outer),
638 _ => None,
639 };
640
641 self.eat_until(b'\n');
642 LineComment { doc_style }
643 }
644
645 fn block_comment(&mut self) -> TokenKind {
646 debug_assert!(self.prev() == '/' && self.first() == '*');
647 self.bump();
648
649 let doc_style = match self.first() {
650 // `/*!` is an inner block doc comment.
651 '!' => Some(DocStyle::Inner),
652 // `/***` (more than 2 stars) is not considered a doc comment.
653 // `/**/` is not considered a doc comment.
654 '*' if !matches!(self.second(), '*' | '/') => Some(DocStyle::Outer),
655 _ => None,
656 };
657
658 let mut depth = 1usize;
659 while let Some(c) = self.bump() {
660 match c {
661 '/' if self.first() == '*' => {
662 self.bump();
663 depth += 1;
664 }
665 '*' if self.first() == '/' => {
666 self.bump();
667 depth -= 1;
668 if depth == 0 {
669 // This block comment is closed, so for a construction like "/* */ */"
670 // there will be a successfully parsed block comment "/* */"
671 // and " */" will be processed separately.
672 break;
673 }
674 }
675 _ => (),
676 }
677 }
678
679 BlockComment { doc_style, terminated: depth == 0 }
680 }
681
682 fn whitespace(&mut self) -> TokenKind {
683 debug_assert!(is_whitespace(self.prev()));
684 self.eat_while(is_whitespace);
685 Whitespace
686 }
687
688 fn raw_ident(&mut self) -> TokenKind {
689 debug_assert!(self.prev() == 'r' && self.first() == '#' && is_id_start(self.second()));
690 // Eat "#" symbol.
691 self.bump();
692 // Eat the identifier part of RawIdent.
693 self.eat_identifier();
694 RawIdent
695 }
696
697 fn ident_or_unknown_prefix(&mut self) -> TokenKind {
698 debug_assert!(is_id_start(self.prev()));
699 // Start is already eaten, eat the rest of identifier.
700 self.eat_while(is_id_continue);
701 // Known prefixes must have been handled earlier. So if
702 // we see a prefix here, it is definitely an unknown prefix.
703 match self.first() {
704 '#' | '"' | '\'' => UnknownPrefix,
705 c if !c.is_ascii() && c.is_emoji_char() => self.invalid_ident(),
706 _ => Ident,
707 }
708 }
709
710 fn invalid_ident(&mut self) -> TokenKind {
711 // Start is already eaten, eat the rest of identifier.
712 self.eat_while(|c| {
713 const ZERO_WIDTH_JOINER: char = '\u{200d}';
714 is_id_continue(c) || (!c.is_ascii() && c.is_emoji_char()) || c == ZERO_WIDTH_JOINER
715 });
716 // An invalid identifier followed by '#' or '"' or '\'' could be
717 // interpreted as an invalid literal prefix. We don't bother doing that
718 // because the treatment of invalid identifiers and invalid prefixes
719 // would be the same.
720 InvalidIdent
721 }
722
723 fn c_or_byte_string(
724 &mut self,
725 mk_kind: fn(bool) -> LiteralKind,
726 mk_kind_raw: fn(Option<u8>) -> LiteralKind,
727 single_quoted: Option<fn(bool) -> LiteralKind>,
728 ) -> TokenKind {
729 match (self.first(), self.second(), single_quoted) {
730 ('\'', _, Some(single_quoted)) => {
731 self.bump();
732 let terminated = self.single_quoted_string();
733 let suffix_start = self.pos_within_token();
734 if terminated {
735 self.eat_literal_suffix();
736 }
737 let kind = single_quoted(terminated);
738 Literal { kind, suffix_start }
739 }
740 ('"', _, _) => {
741 self.bump();
742 let terminated = self.double_quoted_string();
743 let suffix_start = self.pos_within_token();
744 if terminated {
745 self.eat_literal_suffix();
746 }
747 let kind = mk_kind(terminated);
748 Literal { kind, suffix_start }
749 }
750 ('r', '"', _) | ('r', '#', _) => {
751 self.bump();
752 let res = self.raw_double_quoted_string(2);
753 let suffix_start = self.pos_within_token();
754 if res.is_ok() {
755 self.eat_literal_suffix();
756 }
757 let kind = mk_kind_raw(res.ok());
758 Literal { kind, suffix_start }
759 }
760 _ => self.ident_or_unknown_prefix(),
761 }
762 }
763
764 fn number(&mut self, first_digit: char) -> LiteralKind {
765 debug_assert!('0' <= self.prev() && self.prev() <= '9');
766 let mut base = Base::Decimal;
767 if first_digit == '0' {
768 // Attempt to parse encoding base.
769 match self.first() {
770 'b' => {
771 base = Base::Binary;
772 self.bump();
773 if !self.eat_decimal_digits() {
774 return Int { base, empty_int: true };
775 }
776 }
777 'o' => {
778 base = Base::Octal;
779 self.bump();
780 if !self.eat_decimal_digits() {
781 return Int { base, empty_int: true };
782 }
783 }
784 'x' => {
785 base = Base::Hexadecimal;
786 self.bump();
787 if !self.eat_hexadecimal_digits() {
788 return Int { base, empty_int: true };
789 }
790 }
791 // Not a base prefix; consume additional digits.
792 '0'..='9' | '_' => {
793 self.eat_decimal_digits();
794 }
795
796 // Also not a base prefix; nothing more to do here.
797 '.' | 'e' | 'E' => {}
798
799 // Just a 0.
800 _ => return Int { base, empty_int: false },
801 }
802 } else {
803 // No base prefix, parse number in the usual way.
804 self.eat_decimal_digits();
805 }
806
807 match self.first() {
808 // Don't be greedy if this is actually an
809 // integer literal followed by field/method access or a range pattern
810 // (`0..2` and `12.foo()`)
811 '.' if self.second() != '.' && !is_id_start(self.second()) => {
812 // might have stuff after the ., and if it does, it needs to start
813 // with a number
814 self.bump();
815 let mut empty_exponent = false;
816 if self.first().is_ascii_digit() {
817 self.eat_decimal_digits();
818 match self.first() {
819 'e' | 'E' => {
820 self.bump();
821 empty_exponent = !self.eat_float_exponent();
822 }
823 _ => (),
824 }
825 }
826 Float { base, empty_exponent }
827 }
828 'e' | 'E' => {
829 self.bump();
830 let empty_exponent = !self.eat_float_exponent();
831 Float { base, empty_exponent }
832 }
833 _ => Int { base, empty_int: false },
834 }
835 }
836
837 fn lifetime_or_char(&mut self) -> TokenKind {
838 debug_assert!(self.prev() == '\'');
839
840 let can_be_a_lifetime = if self.second() == '\'' {
841 // It's surely not a lifetime.
842 false
843 } else {
844 // If the first symbol is valid for identifier, it can be a lifetime.
845 // Also check if it's a number for a better error reporting (so '0 will
846 // be reported as invalid lifetime and not as unterminated char literal).
847 is_id_start(self.first()) || self.first().is_ascii_digit()
848 };
849
850 if !can_be_a_lifetime {
851 let terminated = self.single_quoted_string();
852 let suffix_start = self.pos_within_token();
853 if terminated {
854 self.eat_literal_suffix();
855 }
856 let kind = Char { terminated };
857 return Literal { kind, suffix_start };
858 }
859
860 if self.first() == 'r' && self.second() == '#' && is_id_start(self.third()) {
861 // Eat "r" and `#`, and identifier start characters.
862 self.bump();
863 self.bump();
864 self.bump();
865 self.eat_while(is_id_continue);
866 return RawLifetime;
867 }
868
869 // Either a lifetime or a character literal with
870 // length greater than 1.
871 let starts_with_number = self.first().is_ascii_digit();
872
873 // Skip the literal contents.
874 // First symbol can be a number (which isn't a valid identifier start),
875 // so skip it without any checks.
876 self.bump();
877 self.eat_while(is_id_continue);
878
879 match self.first() {
880 // Check if after skipping literal contents we've met a closing
881 // single quote (which means that user attempted to create a
882 // string with single quotes).
883 '\'' => {
884 self.bump();
885 let kind = Char { terminated: true };
886 Literal { kind, suffix_start: self.pos_within_token() }
887 }
888 '#' if !starts_with_number => UnknownPrefixLifetime,
889 _ => Lifetime { starts_with_number },
890 }
891 }
892
893 fn single_quoted_string(&mut self) -> bool {
894 debug_assert!(self.prev() == '\'');
895 // Check if it's a one-symbol literal.
896 if self.second() == '\'' && self.first() != '\\' {
897 self.bump();
898 self.bump();
899 return true;
900 }
901
902 // Literal has more than one symbol.
903
904 // Parse until either quotes are terminated or error is detected.
905 loop {
906 match self.first() {
907 // Quotes are terminated, finish parsing.
908 '\'' => {
909 self.bump();
910 return true;
911 }
912 // Probably beginning of the comment, which we don't want to include
913 // to the error report.
914 '/' => break,
915 // Newline without following '\'' means unclosed quote, stop parsing.
916 '\n' if self.second() != '\'' => break,
917 // End of file, stop parsing.
918 EOF_CHAR if self.is_eof() => break,
919 // Escaped slash is considered one character, so bump twice.
920 '\\' => {
921 self.bump();
922 self.bump();
923 }
924 // Skip the character.
925 _ => {
926 self.bump();
927 }
928 }
929 }
930 // String was not terminated.
931 false
932 }
933
934 /// Eats double-quoted string and returns true
935 /// if string is terminated.
936 fn double_quoted_string(&mut self) -> bool {
937 debug_assert!(self.prev() == '"');
938 while let Some(c) = self.bump() {
939 match c {
940 '"' => {
941 return true;
942 }
943 '\\' if self.first() == '\\' || self.first() == '"' => {
944 // Bump again to skip escaped character.
945 self.bump();
946 }
947 _ => (),
948 }
949 }
950 // End of file reached.
951 false
952 }
953
954 /// Attempt to lex for a guarded string literal.
955 ///
956 /// Used by `rustc_parse::lexer` to lex for guarded strings
957 /// conditionally based on edition.
958 ///
959 /// Note: this will not reset the `Cursor` when a
960 /// guarded string is not found. It is the caller's
961 /// responsibility to do so.
962 pub fn guarded_double_quoted_string(&mut self) -> Option<GuardedStr> {
963 debug_assert!(self.prev() != '#');
964
965 let mut n_start_hashes: u32 = 0;
966 while self.first() == '#' {
967 n_start_hashes += 1;
968 self.bump();
969 }
970
971 if self.first() != '"' {
972 return None;
973 }
974 self.bump();
975 debug_assert!(self.prev() == '"');
976
977 // Lex the string itself as a normal string literal
978 // so we can recover that for older editions later.
979 let terminated = self.double_quoted_string();
980 if !terminated {
981 let token_len = self.pos_within_token();
982 self.reset_pos_within_token();
983
984 return Some(GuardedStr { n_hashes: n_start_hashes, terminated: false, token_len });
985 }
986
987 // Consume closing '#' symbols.
988 // Note that this will not consume extra trailing `#` characters:
989 // `###"abcde"####` is lexed as a `GuardedStr { n_end_hashes: 3, .. }`
990 // followed by a `#` token.
991 let mut n_end_hashes = 0;
992 while self.first() == '#' && n_end_hashes < n_start_hashes {
993 n_end_hashes += 1;
994 self.bump();
995 }
996
997 // Reserved syntax, always an error, so it doesn't matter if
998 // `n_start_hashes != n_end_hashes`.
999
1000 self.eat_literal_suffix();
1001
1002 let token_len = self.pos_within_token();
1003 self.reset_pos_within_token();
1004
1005 Some(GuardedStr { n_hashes: n_start_hashes, terminated: true, token_len })
1006 }
1007
1008 /// Eats the double-quoted string and returns `n_hashes` and an error if encountered.
1009 fn raw_double_quoted_string(&mut self, prefix_len: u32) -> Result<u8, RawStrError> {
1010 // Wrap the actual function to handle the error with too many hashes.
1011 // This way, it eats the whole raw string.
1012 let n_hashes = self.raw_string_unvalidated(prefix_len)?;
1013 // Only up to 255 `#`s are allowed in raw strings
1014 match u8::try_from(n_hashes) {
1015 Ok(num) => Ok(num),
1016 Err(_) => Err(RawStrError::TooManyDelimiters { found: n_hashes }),
1017 }
1018 }
1019
1020 fn raw_string_unvalidated(&mut self, prefix_len: u32) -> Result<u32, RawStrError> {
1021 debug_assert!(self.prev() == 'r');
1022 let start_pos = self.pos_within_token();
1023 let mut possible_terminator_offset = None;
1024 let mut max_hashes = 0;
1025
1026 // Count opening '#' symbols.
1027 let mut eaten = 0;
1028 while self.first() == '#' {
1029 eaten += 1;
1030 self.bump();
1031 }
1032 let n_start_hashes = eaten;
1033
1034 // Check that string is started.
1035 match self.bump() {
1036 Some('"') => (),
1037 c => {
1038 let c = c.unwrap_or(EOF_CHAR);
1039 return Err(RawStrError::InvalidStarter { bad_char: c });
1040 }
1041 }
1042
1043 // Skip the string contents and on each '#' character met, check if this is
1044 // a raw string termination.
1045 loop {
1046 self.eat_until(b'"');
1047
1048 if self.is_eof() {
1049 return Err(RawStrError::NoTerminator {
1050 expected: n_start_hashes,
1051 found: max_hashes,
1052 possible_terminator_offset,
1053 });
1054 }
1055
1056 // Eat closing double quote.
1057 self.bump();
1058
1059 // Check that amount of closing '#' symbols
1060 // is equal to the amount of opening ones.
1061 // Note that this will not consume extra trailing `#` characters:
1062 // `r###"abcde"####` is lexed as a `RawStr { n_hashes: 3 }`
1063 // followed by a `#` token.
1064 let mut n_end_hashes = 0;
1065 while self.first() == '#' && n_end_hashes < n_start_hashes {
1066 n_end_hashes += 1;
1067 self.bump();
1068 }
1069
1070 if n_end_hashes == n_start_hashes {
1071 return Ok(n_start_hashes);
1072 } else if n_end_hashes > max_hashes {
1073 // Keep track of possible terminators to give a hint about
1074 // where there might be a missing terminator
1075 possible_terminator_offset =
1076 Some(self.pos_within_token() - start_pos - n_end_hashes + prefix_len);
1077 max_hashes = n_end_hashes;
1078 }
1079 }
1080 }
1081
1082 fn eat_decimal_digits(&mut self) -> bool {
1083 let mut has_digits = false;
1084 loop {
1085 match self.first() {
1086 '_' => {
1087 self.bump();
1088 }
1089 '0'..='9' => {
1090 has_digits = true;
1091 self.bump();
1092 }
1093 _ => break,
1094 }
1095 }
1096 has_digits
1097 }
1098
1099 fn eat_hexadecimal_digits(&mut self) -> bool {
1100 let mut has_digits = false;
1101 loop {
1102 match self.first() {
1103 '_' => {
1104 self.bump();
1105 }
1106 '0'..='9' | 'a'..='f' | 'A'..='F' => {
1107 has_digits = true;
1108 self.bump();
1109 }
1110 _ => break,
1111 }
1112 }
1113 has_digits
1114 }
1115
1116 /// Eats the float exponent. Returns true if at least one digit was met,
1117 /// and returns false otherwise.
1118 fn eat_float_exponent(&mut self) -> bool {
1119 debug_assert!(self.prev() == 'e' || self.prev() == 'E');
1120 if self.first() == '-' || self.first() == '+' {
1121 self.bump();
1122 }
1123 self.eat_decimal_digits()
1124 }
1125
1126 // Eats the suffix of the literal, e.g. "u8".
1127 fn eat_literal_suffix(&mut self) {
1128 self.eat_identifier();
1129 }
1130
1131 // Eats the identifier. Note: succeeds on `_`, which isn't a valid
1132 // identifier.
1133 fn eat_identifier(&mut self) {
1134 if !is_id_start(self.first()) {
1135 return;
1136 }
1137 self.bump();
1138
1139 self.eat_while(is_id_continue);
1140 }
1141}