rustc_parse/parser/
token_type.rs

1use rustc_ast::token::TokenKind;
2use rustc_span::symbol::{Symbol, kw, sym};
3
4/// Used in "expected"/"expected one of" error messages. Tokens are added here
5/// as necessary. Tokens with values (e.g. literals, identifiers) are
6/// represented by a single variant (e.g. `Literal`, `Ident`).
7///
8/// It's an awkward representation, but it's important for performance. It's a
9/// C-style parameterless enum so that `TokenTypeSet` can be a bitset. This is
10/// important because `Parser::expected_token_types` is very hot. `TokenType`
11/// used to have variants with parameters (e.g. all the keywords were in a
12/// single `Keyword` variant with a `Symbol` parameter) and
13/// `Parser::expected_token_types` was a `Vec<TokenType>` which was much slower
14/// to manipulate.
15///
16/// We really want to keep the number of variants to 128 or fewer, so that
17/// `TokenTypeSet` can be implemented with a `u128`.
18#[derive(Debug, Clone, Copy, PartialEq)]
19pub enum TokenType {
20    // Expression-operator symbols
21    Eq,
22    Lt,
23    Le,
24    EqEq,
25    Gt,
26    AndAnd,
27    OrOr,
28    Bang,
29    Tilde,
30
31    // BinOps
32    Plus,
33    Minus,
34    Star,
35    And,
36    Or,
37
38    // Structural symbols
39    At,
40    Dot,
41    DotDot,
42    DotDotDot,
43    DotDotEq,
44    Comma,
45    Semi,
46    Colon,
47    PathSep,
48    RArrow,
49    FatArrow,
50    Pound,
51    Question,
52    OpenParen,
53    CloseParen,
54    OpenBrace,
55    CloseBrace,
56    OpenBracket,
57    CloseBracket,
58    Eof,
59
60    // Token types with some details elided.
61    /// Any operator.
62    Operator,
63    /// Any identifier token.
64    Ident,
65    /// Any lifetime token.
66    Lifetime,
67    /// Any token that can start a path.
68    Path,
69    /// Any token that can start a type.
70    Type,
71    /// Any token that can start a const expression.
72    Const,
73
74    // Keywords
75    // tidy-alphabetical-start
76    KwAs,
77    KwAsync,
78    KwAuto,
79    KwAwait,
80    KwBecome,
81    KwBox,
82    KwBreak,
83    KwCatch,
84    KwConst,
85    KwContinue,
86    KwContractEnsures,
87    KwContractRequires,
88    KwCrate,
89    KwDefault,
90    KwDyn,
91    KwElse,
92    KwEnum,
93    KwExtern,
94    KwFn,
95    KwFor,
96    KwGen,
97    KwIf,
98    KwImpl,
99    KwIn,
100    KwLet,
101    KwLoop,
102    KwMacro,
103    KwMacroRules,
104    KwMatch,
105    KwMod,
106    KwMove,
107    KwMut,
108    KwPub,
109    KwRaw,
110    KwRef,
111    KwReturn,
112    KwReuse,
113    KwSafe,
114    KwSelfUpper,
115    KwStatic,
116    KwStruct,
117    KwSuper,
118    KwTrait,
119    KwTry,
120    KwType,
121    KwUnderscore,
122    KwUnsafe,
123    KwUse,
124    KwWhere,
125    KwWhile,
126    KwYield,
127    // tidy-alphabetical-end
128
129    // Keyword-like symbols.
130    // tidy-alphabetical-start
131    SymAttSyntax,
132    SymClobberAbi,
133    SymInlateout,
134    SymInout,
135    SymIs,
136    SymLabel,
137    SymLateout,
138    SymMayUnwind,
139    SymNomem,
140    SymNoreturn,
141    SymNostack,
142    SymNull,
143    SymOptions,
144    SymOut,
145    SymPreservesFlags,
146    SymPure,
147    SymReadonly,
148    SymSym,
149    // tidy-alphabetical-end
150}
151
152// Macro to avoid repetitive boilerplate code.
153macro_rules! from_u32_match {
154    ($val:ident; $($tok:ident,)+) => {
155        // A more obvious formulation would be `0 => TokenType::Eq`. But
156        // this formulation with the guard lets us avoid specifying a
157        // specific integer for each variant.
158        match $val {
159            $(
160                t if t == TokenType::$tok as u32 => TokenType::$tok,
161            )+
162            _ => panic!("unhandled value: {}", $val),
163        }
164    };
165}
166
167impl TokenType {
168    fn from_u32(val: u32) -> TokenType {
169        let token_type = from_u32_match! { val;
170            Eq,
171            Lt,
172            Le,
173            EqEq,
174            Gt,
175            AndAnd,
176            OrOr,
177            Bang,
178            Tilde,
179
180            Plus,
181            Minus,
182            Star,
183            And,
184            Or,
185
186            At,
187            Dot,
188            DotDot,
189            DotDotDot,
190            DotDotEq,
191            Comma,
192            Semi,
193            Colon,
194            PathSep,
195            RArrow,
196            FatArrow,
197            Pound,
198            Question,
199            OpenParen,
200            CloseParen,
201            OpenBrace,
202            CloseBrace,
203            OpenBracket,
204            CloseBracket,
205            Eof,
206
207            Operator,
208            Ident,
209            Lifetime,
210            Path,
211            Type,
212            Const,
213
214            KwAs,
215            KwAsync,
216            KwAuto,
217            KwAwait,
218            KwBecome,
219            KwBox,
220            KwBreak,
221            KwCatch,
222            KwConst,
223            KwContinue,
224            KwContractEnsures,
225            KwContractRequires,
226            KwCrate,
227            KwDefault,
228            KwDyn,
229            KwElse,
230            KwEnum,
231            KwExtern,
232            KwFn,
233            KwFor,
234            KwGen,
235            KwIf,
236            KwImpl,
237            KwIn,
238            KwLet,
239            KwLoop,
240            KwMacro,
241            KwMacroRules,
242            KwMatch,
243            KwMod,
244            KwMove,
245            KwMut,
246            KwPub,
247            KwRaw,
248            KwRef,
249            KwReturn,
250            KwReuse,
251            KwSafe,
252            KwSelfUpper,
253            KwStatic,
254            KwStruct,
255            KwSuper,
256            KwTrait,
257            KwTry,
258            KwType,
259            KwUnderscore,
260            KwUnsafe,
261            KwUse,
262            KwWhere,
263            KwWhile,
264            KwYield,
265
266            SymAttSyntax,
267            SymClobberAbi,
268            SymInlateout,
269            SymInout,
270            SymIs,
271            SymLabel,
272            SymLateout,
273            SymMayUnwind,
274            SymNomem,
275            SymNoreturn,
276            SymNostack,
277            SymNull,
278            SymOptions,
279            SymOut,
280            SymPreservesFlags,
281            SymPure,
282            SymReadonly,
283            SymSym,
284        };
285        token_type
286    }
287
288    pub(super) fn is_keyword(&self) -> Option<Symbol> {
289        match self {
290            TokenType::KwAs => Some(kw::As),
291            TokenType::KwAsync => Some(kw::Async),
292            TokenType::KwAuto => Some(kw::Auto),
293            TokenType::KwAwait => Some(kw::Await),
294            TokenType::KwBecome => Some(kw::Become),
295            TokenType::KwBox => Some(kw::Box),
296            TokenType::KwBreak => Some(kw::Break),
297            TokenType::KwCatch => Some(kw::Catch),
298            TokenType::KwConst => Some(kw::Const),
299            TokenType::KwContinue => Some(kw::Continue),
300            TokenType::KwContractEnsures => Some(kw::ContractEnsures),
301            TokenType::KwContractRequires => Some(kw::ContractRequires),
302            TokenType::KwCrate => Some(kw::Crate),
303            TokenType::KwDefault => Some(kw::Default),
304            TokenType::KwDyn => Some(kw::Dyn),
305            TokenType::KwElse => Some(kw::Else),
306            TokenType::KwEnum => Some(kw::Enum),
307            TokenType::KwExtern => Some(kw::Extern),
308            TokenType::KwFn => Some(kw::Fn),
309            TokenType::KwFor => Some(kw::For),
310            TokenType::KwGen => Some(kw::Gen),
311            TokenType::KwIf => Some(kw::If),
312            TokenType::KwImpl => Some(kw::Impl),
313            TokenType::KwIn => Some(kw::In),
314            TokenType::KwLet => Some(kw::Let),
315            TokenType::KwLoop => Some(kw::Loop),
316            TokenType::KwMacroRules => Some(kw::MacroRules),
317            TokenType::KwMacro => Some(kw::Macro),
318            TokenType::KwMatch => Some(kw::Match),
319            TokenType::KwMod => Some(kw::Mod),
320            TokenType::KwMove => Some(kw::Move),
321            TokenType::KwMut => Some(kw::Mut),
322            TokenType::KwPub => Some(kw::Pub),
323            TokenType::KwRaw => Some(kw::Raw),
324            TokenType::KwRef => Some(kw::Ref),
325            TokenType::KwReturn => Some(kw::Return),
326            TokenType::KwReuse => Some(kw::Reuse),
327            TokenType::KwSafe => Some(kw::Safe),
328            TokenType::KwSelfUpper => Some(kw::SelfUpper),
329            TokenType::KwStatic => Some(kw::Static),
330            TokenType::KwStruct => Some(kw::Struct),
331            TokenType::KwSuper => Some(kw::Super),
332            TokenType::KwTrait => Some(kw::Trait),
333            TokenType::KwTry => Some(kw::Try),
334            TokenType::KwType => Some(kw::Type),
335            TokenType::KwUnderscore => Some(kw::Underscore),
336            TokenType::KwUnsafe => Some(kw::Unsafe),
337            TokenType::KwUse => Some(kw::Use),
338            TokenType::KwWhere => Some(kw::Where),
339            TokenType::KwWhile => Some(kw::While),
340            TokenType::KwYield => Some(kw::Yield),
341
342            TokenType::SymAttSyntax => Some(sym::att_syntax),
343            TokenType::SymClobberAbi => Some(sym::clobber_abi),
344            TokenType::SymInlateout => Some(sym::inlateout),
345            TokenType::SymInout => Some(sym::inout),
346            TokenType::SymIs => Some(sym::is),
347            TokenType::SymLabel => Some(sym::label),
348            TokenType::SymLateout => Some(sym::lateout),
349            TokenType::SymMayUnwind => Some(sym::may_unwind),
350            TokenType::SymNomem => Some(sym::nomem),
351            TokenType::SymNoreturn => Some(sym::noreturn),
352            TokenType::SymNostack => Some(sym::nostack),
353            TokenType::SymNull => Some(sym::null),
354            TokenType::SymOptions => Some(sym::options),
355            TokenType::SymOut => Some(sym::out),
356            TokenType::SymPreservesFlags => Some(sym::preserves_flags),
357            TokenType::SymPure => Some(sym::pure),
358            TokenType::SymReadonly => Some(sym::readonly),
359            TokenType::SymSym => Some(sym::sym),
360            _ => None,
361        }
362    }
363
364    // The output should be the same as that produced by
365    // `rustc_ast_pretty::pprust::token_to_string`.
366    pub(super) fn to_string(&self) -> String {
367        match self {
368            TokenType::Eq => "`=`",
369            TokenType::Lt => "`<`",
370            TokenType::Le => "`<=`",
371            TokenType::EqEq => "`==`",
372            TokenType::Gt => "`>`",
373            TokenType::AndAnd => "`&&`",
374            TokenType::OrOr => "`||`",
375            TokenType::Bang => "`!`",
376            TokenType::Tilde => "`~`",
377
378            TokenType::Plus => "`+`",
379            TokenType::Minus => "`-`",
380            TokenType::Star => "`*`",
381            TokenType::And => "`&`",
382            TokenType::Or => "`|`",
383
384            TokenType::At => "`@`",
385            TokenType::Dot => "`.`",
386            TokenType::DotDot => "`..`",
387            TokenType::DotDotDot => "`...`",
388            TokenType::DotDotEq => "`..=`",
389            TokenType::Comma => "`,`",
390            TokenType::Semi => "`;`",
391            TokenType::Colon => "`:`",
392            TokenType::PathSep => "`::`",
393            TokenType::RArrow => "`->`",
394            TokenType::FatArrow => "`=>`",
395            TokenType::Pound => "`#`",
396            TokenType::Question => "`?`",
397            TokenType::OpenParen => "`(`",
398            TokenType::CloseParen => "`)`",
399            TokenType::OpenBrace => "`{`",
400            TokenType::CloseBrace => "`}`",
401            TokenType::OpenBracket => "`[`",
402            TokenType::CloseBracket => "`]`",
403            TokenType::Eof => "<eof>",
404
405            TokenType::Operator => "an operator",
406            TokenType::Ident => "identifier",
407            TokenType::Lifetime => "lifetime",
408            TokenType::Path => "path",
409            TokenType::Type => "type",
410            TokenType::Const => "a const expression",
411
412            _ => return format!("`{}`", self.is_keyword().unwrap()),
413        }
414        .to_string()
415    }
416}
417
418/// Used by various `Parser` methods such as `check` and `eat`. The first field
419/// is always by used those methods. The second field is only used when the
420/// first field doesn't match.
421#[derive(Clone, Copy, Debug)]
422pub struct ExpTokenPair {
423    pub tok: TokenKind,
424    pub token_type: TokenType,
425}
426
427/// Used by various `Parser` methods such as `check_keyword` and `eat_keyword`.
428/// The first field is always used by those methods. The second field is only
429/// used when the first field doesn't match.
430#[derive(Clone, Copy)]
431pub struct ExpKeywordPair {
432    pub kw: Symbol,
433    pub token_type: TokenType,
434}
435
436// Gets a statically-known `ExpTokenPair` pair (for non-keywords) or
437// `ExpKeywordPair` (for keywords), as used with various `check`/`expect`
438// methods in `Parser`.
439//
440// The name is short because it's used a lot.
441#[macro_export]
442// We don't use the normal `#[rustfmt::skip]` here because that triggers a
443// bogus "macro-expanded `macro_export` macros from the current crate cannot be
444// referred to by absolute paths" error, ugh. See #52234.
445#[cfg_attr(rustfmt, rustfmt::skip)]
446macro_rules! exp {
447    // `ExpTokenPair` helper rules.
448    (@tok, $tok:ident) => {
449        $crate::parser::token_type::ExpTokenPair {
450            tok: rustc_ast::token::$tok,
451            token_type: $crate::parser::token_type::TokenType::$tok
452        }
453    };
454
455    // `ExpKeywordPair` helper rules.
456    (@kw, $kw:ident, $token_type:ident) => {
457        $crate::parser::token_type::ExpKeywordPair {
458            kw: rustc_span::symbol::kw::$kw,
459            token_type: $crate::parser::token_type::TokenType::$token_type,
460        }
461    };
462    (@sym, $kw:ident, $token_type:ident) => {
463        $crate::parser::token_type::ExpKeywordPair {
464            kw: rustc_span::symbol::sym::$kw,
465            token_type: $crate::parser::token_type::TokenType::$token_type,
466        }
467    };
468
469    (Eq)             => { exp!(@tok, Eq) };
470    (Lt)             => { exp!(@tok, Lt) };
471    (Le)             => { exp!(@tok, Le) };
472    (EqEq)           => { exp!(@tok, EqEq) };
473    (Gt)             => { exp!(@tok, Gt) };
474    (AndAnd)         => { exp!(@tok, AndAnd) };
475    (OrOr)           => { exp!(@tok, OrOr) };
476    (Bang)           => { exp!(@tok, Bang) };
477    (Tilde)          => { exp!(@tok, Tilde) };
478    (Plus)           => { exp!(@tok, Plus) };
479    (Minus)          => { exp!(@tok, Minus) };
480    (Star)           => { exp!(@tok, Star) };
481    (And)            => { exp!(@tok, And) };
482    (Or)             => { exp!(@tok, Or) };
483    (At)             => { exp!(@tok, At) };
484    (Dot)            => { exp!(@tok, Dot) };
485    (DotDot)         => { exp!(@tok, DotDot) };
486    (DotDotDot)      => { exp!(@tok, DotDotDot) };
487    (DotDotEq)       => { exp!(@tok, DotDotEq) };
488    (Comma)          => { exp!(@tok, Comma) };
489    (Semi)           => { exp!(@tok, Semi) };
490    (Colon)          => { exp!(@tok, Colon) };
491    (PathSep)        => { exp!(@tok, PathSep) };
492    (RArrow)         => { exp!(@tok, RArrow) };
493    (FatArrow)       => { exp!(@tok, FatArrow) };
494    (Pound)          => { exp!(@tok, Pound) };
495    (Question)       => { exp!(@tok, Question) };
496    (Eof)            => { exp!(@tok, Eof) };
497
498    (OpenParen)      => { exp!(@tok, OpenParen) };
499    (OpenBrace)      => { exp!(@tok, OpenBrace) };
500    (OpenBracket)    => { exp!(@tok, OpenBracket) };
501    (CloseParen)     => { exp!(@tok, CloseParen) };
502    (CloseBrace)     => { exp!(@tok, CloseBrace) };
503    (CloseBracket)   => { exp!(@tok, CloseBracket) };
504
505    (As)             => { exp!(@kw, As,         KwAs) };
506    (Async)          => { exp!(@kw, Async,      KwAsync) };
507    (Auto)           => { exp!(@kw, Auto,       KwAuto) };
508    (Await)          => { exp!(@kw, Await,      KwAwait) };
509    (Become)         => { exp!(@kw, Become,     KwBecome) };
510    (Box)            => { exp!(@kw, Box,        KwBox) };
511    (Break)          => { exp!(@kw, Break,      KwBreak) };
512    (Catch)          => { exp!(@kw, Catch,      KwCatch) };
513    (Const)          => { exp!(@kw, Const,      KwConst) };
514    (Continue)       => { exp!(@kw, Continue,   KwContinue) };
515    (ContractEnsures)  => { exp!(@kw, ContractEnsures, KwContractEnsures) };
516    (ContractRequires) => { exp!(@kw, ContractRequires, KwContractRequires) };
517    (Crate)          => { exp!(@kw, Crate,      KwCrate) };
518    (Default)        => { exp!(@kw, Default,    KwDefault) };
519    (Dyn)            => { exp!(@kw, Dyn,        KwDyn) };
520    (Else)           => { exp!(@kw, Else,       KwElse) };
521    (Enum)           => { exp!(@kw, Enum,       KwEnum) };
522    (Extern)         => { exp!(@kw, Extern,     KwExtern) };
523    (Fn)             => { exp!(@kw, Fn,         KwFn) };
524    (For)            => { exp!(@kw, For,        KwFor) };
525    (Gen)            => { exp!(@kw, Gen,        KwGen) };
526    (If)             => { exp!(@kw, If,         KwIf) };
527    (Impl)           => { exp!(@kw, Impl,       KwImpl) };
528    (In)             => { exp!(@kw, In,         KwIn) };
529    (Let)            => { exp!(@kw, Let,        KwLet) };
530    (Loop)           => { exp!(@kw, Loop,       KwLoop) };
531    (Macro)          => { exp!(@kw, Macro,      KwMacro) };
532    (MacroRules)     => { exp!(@kw, MacroRules, KwMacroRules) };
533    (Match)          => { exp!(@kw, Match,      KwMatch) };
534    (Mod)            => { exp!(@kw, Mod,        KwMod) };
535    (Move)           => { exp!(@kw, Move,       KwMove) };
536    (Mut)            => { exp!(@kw, Mut,        KwMut) };
537    (Pub)            => { exp!(@kw, Pub,        KwPub) };
538    (Raw)            => { exp!(@kw, Raw,        KwRaw) };
539    (Ref)            => { exp!(@kw, Ref,        KwRef) };
540    (Return)         => { exp!(@kw, Return,     KwReturn) };
541    (Reuse)          => { exp!(@kw, Reuse,      KwReuse) };
542    (Safe)           => { exp!(@kw, Safe,       KwSafe) };
543    (SelfUpper)      => { exp!(@kw, SelfUpper,  KwSelfUpper) };
544    (Static)         => { exp!(@kw, Static,     KwStatic) };
545    (Struct)         => { exp!(@kw, Struct,     KwStruct) };
546    (Super)          => { exp!(@kw, Super,      KwSuper) };
547    (Trait)          => { exp!(@kw, Trait,      KwTrait) };
548    (Try)            => { exp!(@kw, Try,        KwTry) };
549    (Type)           => { exp!(@kw, Type,       KwType) };
550    (Underscore)     => { exp!(@kw, Underscore, KwUnderscore) };
551    (Unsafe)         => { exp!(@kw, Unsafe,     KwUnsafe) };
552    (Use)            => { exp!(@kw, Use,        KwUse) };
553    (Where)          => { exp!(@kw, Where,      KwWhere) };
554    (While)          => { exp!(@kw, While,      KwWhile) };
555    (Yield)          => { exp!(@kw, Yield,      KwYield) };
556
557    (AttSyntax)      => { exp!(@sym, att_syntax,      SymAttSyntax) };
558    (ClobberAbi)     => { exp!(@sym, clobber_abi,     SymClobberAbi) };
559    (Inlateout)      => { exp!(@sym, inlateout,       SymInlateout) };
560    (Inout)          => { exp!(@sym, inout,           SymInout) };
561    (Is)             => { exp!(@sym, is,              SymIs) };
562    (Label)          => { exp!(@sym, label,           SymLabel) };
563    (Lateout)        => { exp!(@sym, lateout,         SymLateout) };
564    (MayUnwind)      => { exp!(@sym, may_unwind,      SymMayUnwind) };
565    (Nomem)          => { exp!(@sym, nomem,           SymNomem) };
566    (Noreturn)       => { exp!(@sym, noreturn,        SymNoreturn) };
567    (Nostack)        => { exp!(@sym, nostack,         SymNostack) };
568    (Null)           => { exp!(@sym, null,            SymNull) };
569    (Options)        => { exp!(@sym, options,         SymOptions) };
570    (Out)            => { exp!(@sym, out,             SymOut) };
571    (PreservesFlags) => { exp!(@sym, preserves_flags, SymPreservesFlags) };
572    (Pure)           => { exp!(@sym, pure,            SymPure) };
573    (Readonly)       => { exp!(@sym, readonly,        SymReadonly) };
574    (Sym)            => { exp!(@sym, sym,             SymSym) };
575}
576
577/// A bitset type designed specifically for `Parser::expected_token_types`,
578/// which is very hot. `u128` is the smallest integer that will fit every
579/// `TokenType` value.
580#[derive(Clone, Copy)]
581pub(super) struct TokenTypeSet(u128);
582
583impl TokenTypeSet {
584    pub(super) fn new() -> TokenTypeSet {
585        TokenTypeSet(0)
586    }
587
588    pub(super) fn is_empty(&self) -> bool {
589        self.0 == 0
590    }
591
592    pub(super) fn insert(&mut self, token_type: TokenType) {
593        self.0 = self.0 | (1u128 << token_type as u32)
594    }
595
596    pub(super) fn clear(&mut self) {
597        self.0 = 0
598    }
599
600    pub(super) fn contains(&self, token_type: TokenType) -> bool {
601        self.0 & (1u128 << token_type as u32) != 0
602    }
603
604    pub(super) fn iter(&self) -> TokenTypeSetIter {
605        TokenTypeSetIter(*self)
606    }
607}
608
609// The `TokenTypeSet` is a copy of the set being iterated. It initially holds
610// the entire set. Each bit is cleared as it is returned. We have finished once
611// it is all zeroes.
612pub(super) struct TokenTypeSetIter(TokenTypeSet);
613
614impl Iterator for TokenTypeSetIter {
615    type Item = TokenType;
616
617    fn next(&mut self) -> Option<TokenType> {
618        let num_bits: u32 = (size_of_val(&self.0.0) * 8) as u32;
619        assert_eq!(num_bits, 128);
620        let z = self.0.0.trailing_zeros();
621        if z == num_bits {
622            None
623        } else {
624            self.0.0 &= !(1 << z); // clear the trailing 1 bit
625            Some(TokenType::from_u32(z))
626        }
627    }
628}
rustc_parse/parser/token_type.rs

rustc_parse/parser/
token_type.rs
