rustc_parse/parser/
token_type.rs

1use rustc_ast::token::TokenKind;
2use rustc_span::symbol::{Symbol, kw, sym};
3
4/// Used in "expected"/"expected one of" error messages. Tokens are added here
5/// as necessary. Tokens with values (e.g. literals, identifiers) are
6/// represented by a single variant (e.g. `Literal`, `Ident`).
7///
8/// It's an awkward representation, but it's important for performance. It's a
9/// C-style parameterless enum so that `TokenTypeSet` can be a bitset. This is
10/// important because `Parser::expected_token_types` is very hot. `TokenType`
11/// used to have variants with parameters (e.g. all the keywords were in a
12/// single `Keyword` variant with a `Symbol` parameter) and
13/// `Parser::expected_token_types` was a `Vec<TokenType>` which was much slower
14/// to manipulate.
15///
16/// We really want to keep the number of variants to 128 or fewer, so that
17/// `TokenTypeSet` can be implemented with a `u128`.
18#[derive(#[automatically_derived]
impl ::core::fmt::Debug for TokenType {
    #[inline]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::write_str(f,
            match self {
                TokenType::Eq => "Eq",
                TokenType::Lt => "Lt",
                TokenType::Le => "Le",
                TokenType::EqEq => "EqEq",
                TokenType::Gt => "Gt",
                TokenType::AndAnd => "AndAnd",
                TokenType::OrOr => "OrOr",
                TokenType::Bang => "Bang",
                TokenType::Tilde => "Tilde",
                TokenType::Plus => "Plus",
                TokenType::Minus => "Minus",
                TokenType::Star => "Star",
                TokenType::And => "And",
                TokenType::Or => "Or",
                TokenType::At => "At",
                TokenType::Dot => "Dot",
                TokenType::DotDot => "DotDot",
                TokenType::DotDotDot => "DotDotDot",
                TokenType::DotDotEq => "DotDotEq",
                TokenType::Comma => "Comma",
                TokenType::Semi => "Semi",
                TokenType::Colon => "Colon",
                TokenType::PathSep => "PathSep",
                TokenType::RArrow => "RArrow",
                TokenType::FatArrow => "FatArrow",
                TokenType::Pound => "Pound",
                TokenType::Question => "Question",
                TokenType::OpenParen => "OpenParen",
                TokenType::CloseParen => "CloseParen",
                TokenType::OpenBrace => "OpenBrace",
                TokenType::CloseBrace => "CloseBrace",
                TokenType::OpenBracket => "OpenBracket",
                TokenType::CloseBracket => "CloseBracket",
                TokenType::Eof => "Eof",
                TokenType::Operator => "Operator",
                TokenType::Ident => "Ident",
                TokenType::Lifetime => "Lifetime",
                TokenType::Path => "Path",
                TokenType::Type => "Type",
                TokenType::Const => "Const",
                TokenType::KwAs => "KwAs",
                TokenType::KwAsync => "KwAsync",
                TokenType::KwAuto => "KwAuto",
                TokenType::KwAwait => "KwAwait",
                TokenType::KwBecome => "KwBecome",
                TokenType::KwBox => "KwBox",
                TokenType::KwBreak => "KwBreak",
                TokenType::KwCatch => "KwCatch",
                TokenType::KwConst => "KwConst",
                TokenType::KwContinue => "KwContinue",
                TokenType::KwContractEnsures => "KwContractEnsures",
                TokenType::KwContractRequires => "KwContractRequires",
                TokenType::KwCrate => "KwCrate",
                TokenType::KwDefault => "KwDefault",
                TokenType::KwDyn => "KwDyn",
                TokenType::KwElse => "KwElse",
                TokenType::KwEnum => "KwEnum",
                TokenType::KwExtern => "KwExtern",
                TokenType::KwFinal => "KwFinal",
                TokenType::KwFn => "KwFn",
                TokenType::KwFor => "KwFor",
                TokenType::KwGen => "KwGen",
                TokenType::KwIf => "KwIf",
                TokenType::KwImpl => "KwImpl",
                TokenType::KwIn => "KwIn",
                TokenType::KwLet => "KwLet",
                TokenType::KwLoop => "KwLoop",
                TokenType::KwMacro => "KwMacro",
                TokenType::KwMacroRules => "KwMacroRules",
                TokenType::KwMatch => "KwMatch",
                TokenType::KwMod => "KwMod",
                TokenType::KwMove => "KwMove",
                TokenType::KwMut => "KwMut",
                TokenType::KwPub => "KwPub",
                TokenType::KwRaw => "KwRaw",
                TokenType::KwRef => "KwRef",
                TokenType::KwReturn => "KwReturn",
                TokenType::KwReuse => "KwReuse",
                TokenType::KwSafe => "KwSafe",
                TokenType::KwSelfUpper => "KwSelfUpper",
                TokenType::KwStatic => "KwStatic",
                TokenType::KwStruct => "KwStruct",
                TokenType::KwSuper => "KwSuper",
                TokenType::KwTrait => "KwTrait",
                TokenType::KwTry => "KwTry",
                TokenType::KwType => "KwType",
                TokenType::KwUnderscore => "KwUnderscore",
                TokenType::KwUnsafe => "KwUnsafe",
                TokenType::KwUse => "KwUse",
                TokenType::KwWhere => "KwWhere",
                TokenType::KwWhile => "KwWhile",
                TokenType::KwYield => "KwYield",
                TokenType::SymAttSyntax => "SymAttSyntax",
                TokenType::SymBikeshed => "SymBikeshed",
                TokenType::SymClobberAbi => "SymClobberAbi",
                TokenType::SymInlateout => "SymInlateout",
                TokenType::SymInout => "SymInout",
                TokenType::SymIs => "SymIs",
                TokenType::SymLabel => "SymLabel",
                TokenType::SymLateout => "SymLateout",
                TokenType::SymMayUnwind => "SymMayUnwind",
                TokenType::SymNomem => "SymNomem",
                TokenType::SymNoreturn => "SymNoreturn",
                TokenType::SymNostack => "SymNostack",
                TokenType::SymNull => "SymNull",
                TokenType::SymOptions => "SymOptions",
                TokenType::SymOut => "SymOut",
                TokenType::SymPin => "SymPin",
                TokenType::SymPreservesFlags => "SymPreservesFlags",
                TokenType::SymPure => "SymPure",
                TokenType::SymReadonly => "SymReadonly",
                TokenType::SymSym => "SymSym",
            })
    }
}Debug, #[automatically_derived]
impl ::core::clone::Clone for TokenType {
    #[inline]
    fn clone(&self) -> TokenType { *self }
}Clone, #[automatically_derived]
impl ::core::marker::Copy for TokenType { }Copy, #[automatically_derived]
impl ::core::cmp::PartialEq for TokenType {
    #[inline]
    fn eq(&self, other: &TokenType) -> bool {
        let __self_discr = ::core::intrinsics::discriminant_value(self);
        let __arg1_discr = ::core::intrinsics::discriminant_value(other);
        __self_discr == __arg1_discr
    }
}PartialEq)]
19pub enum TokenType {
20    // Expression-operator symbols
21    Eq,
22    Lt,
23    Le,
24    EqEq,
25    Gt,
26    AndAnd,
27    OrOr,
28    Bang,
29    Tilde,
30
31    // BinOps
32    Plus,
33    Minus,
34    Star,
35    And,
36    Or,
37
38    // Structural symbols
39    At,
40    Dot,
41    DotDot,
42    DotDotDot,
43    DotDotEq,
44    Comma,
45    Semi,
46    Colon,
47    PathSep,
48    RArrow,
49    FatArrow,
50    Pound,
51    Question,
52    OpenParen,
53    CloseParen,
54    OpenBrace,
55    CloseBrace,
56    OpenBracket,
57    CloseBracket,
58    Eof,
59
60    // Token types with some details elided.
61    /// Any operator.
62    Operator,
63    /// Any identifier token.
64    Ident,
65    /// Any lifetime token.
66    Lifetime,
67    /// Any token that can start a path.
68    Path,
69    /// Any token that can start a type.
70    Type,
71    /// Any token that can start a const expression.
72    Const,
73
74    // Keywords
75    // tidy-alphabetical-start
76    KwAs,
77    KwAsync,
78    KwAuto,
79    KwAwait,
80    KwBecome,
81    KwBox,
82    KwBreak,
83    KwCatch,
84    KwConst,
85    KwContinue,
86    KwContractEnsures,
87    KwContractRequires,
88    KwCrate,
89    KwDefault,
90    KwDyn,
91    KwElse,
92    KwEnum,
93    KwExtern,
94    KwFinal,
95    KwFn,
96    KwFor,
97    KwGen,
98    KwIf,
99    KwImpl,
100    KwIn,
101    KwLet,
102    KwLoop,
103    KwMacro,
104    KwMacroRules,
105    KwMatch,
106    KwMod,
107    KwMove,
108    KwMut,
109    KwPub,
110    KwRaw,
111    KwRef,
112    KwReturn,
113    KwReuse,
114    KwSafe,
115    KwSelfUpper,
116    KwStatic,
117    KwStruct,
118    KwSuper,
119    KwTrait,
120    KwTry,
121    KwType,
122    KwUnderscore,
123    KwUnsafe,
124    KwUse,
125    KwWhere,
126    KwWhile,
127    KwYield,
128    // tidy-alphabetical-end
129
130    // Keyword-like symbols.
131    // tidy-alphabetical-start
132    SymAttSyntax,
133    SymBikeshed,
134    SymClobberAbi,
135    SymInlateout,
136    SymInout,
137    SymIs,
138    SymLabel,
139    SymLateout,
140    SymMayUnwind,
141    SymNomem,
142    SymNoreturn,
143    SymNostack,
144    SymNull,
145    SymOptions,
146    SymOut,
147    SymPin,
148    SymPreservesFlags,
149    SymPure,
150    SymReadonly,
151    SymSym,
152    // tidy-alphabetical-end
153}
154
155// Macro to avoid repetitive boilerplate code.
156macro_rules! from_u32_match {
157    ($val:ident; $($tok:ident,)+) => {
158        // A more obvious formulation would be `0 => TokenType::Eq`. But
159        // this formulation with the guard lets us avoid specifying a
160        // specific integer for each variant.
161        match $val {
162            $(
163                t if t == TokenType::$tok as u32 => TokenType::$tok,
164            )+
165            _ => panic!("unhandled value: {}", $val),
166        }
167    };
168}
169
170impl TokenType {
171    fn from_u32(val: u32) -> TokenType {
172        let token_type = match val {
    t if t == TokenType::Eq as u32 => TokenType::Eq,
    t if t == TokenType::Lt as u32 => TokenType::Lt,
    t if t == TokenType::Le as u32 => TokenType::Le,
    t if t == TokenType::EqEq as u32 => TokenType::EqEq,
    t if t == TokenType::Gt as u32 => TokenType::Gt,
    t if t == TokenType::AndAnd as u32 => TokenType::AndAnd,
    t if t == TokenType::OrOr as u32 => TokenType::OrOr,
    t if t == TokenType::Bang as u32 => TokenType::Bang,
    t if t == TokenType::Tilde as u32 => TokenType::Tilde,
    t if t == TokenType::Plus as u32 => TokenType::Plus,
    t if t == TokenType::Minus as u32 => TokenType::Minus,
    t if t == TokenType::Star as u32 => TokenType::Star,
    t if t == TokenType::And as u32 => TokenType::And,
    t if t == TokenType::Or as u32 => TokenType::Or,
    t if t == TokenType::At as u32 => TokenType::At,
    t if t == TokenType::Dot as u32 => TokenType::Dot,
    t if t == TokenType::DotDot as u32 => TokenType::DotDot,
    t if t == TokenType::DotDotDot as u32 => TokenType::DotDotDot,
    t if t == TokenType::DotDotEq as u32 => TokenType::DotDotEq,
    t if t == TokenType::Comma as u32 => TokenType::Comma,
    t if t == TokenType::Semi as u32 => TokenType::Semi,
    t if t == TokenType::Colon as u32 => TokenType::Colon,
    t if t == TokenType::PathSep as u32 => TokenType::PathSep,
    t if t == TokenType::RArrow as u32 => TokenType::RArrow,
    t if t == TokenType::FatArrow as u32 => TokenType::FatArrow,
    t if t == TokenType::Pound as u32 => TokenType::Pound,
    t if t == TokenType::Question as u32 => TokenType::Question,
    t if t == TokenType::OpenParen as u32 => TokenType::OpenParen,
    t if t == TokenType::CloseParen as u32 => TokenType::CloseParen,
    t if t == TokenType::OpenBrace as u32 => TokenType::OpenBrace,
    t if t == TokenType::CloseBrace as u32 => TokenType::CloseBrace,
    t if t == TokenType::OpenBracket as u32 => TokenType::OpenBracket,
    t if t == TokenType::CloseBracket as u32 => TokenType::CloseBracket,
    t if t == TokenType::Eof as u32 => TokenType::Eof,
    t if t == TokenType::Operator as u32 => TokenType::Operator,
    t if t == TokenType::Ident as u32 => TokenType::Ident,
    t if t == TokenType::Lifetime as u32 => TokenType::Lifetime,
    t if t == TokenType::Path as u32 => TokenType::Path,
    t if t == TokenType::Type as u32 => TokenType::Type,
    t if t == TokenType::Const as u32 => TokenType::Const,
    t if t == TokenType::KwAs as u32 => TokenType::KwAs,
    t if t == TokenType::KwAsync as u32 => TokenType::KwAsync,
    t if t == TokenType::KwAuto as u32 => TokenType::KwAuto,
    t if t == TokenType::KwAwait as u32 => TokenType::KwAwait,
    t if t == TokenType::KwBecome as u32 => TokenType::KwBecome,
    t if t == TokenType::KwBox as u32 => TokenType::KwBox,
    t if t == TokenType::KwBreak as u32 => TokenType::KwBreak,
    t if t == TokenType::KwCatch as u32 => TokenType::KwCatch,
    t if t == TokenType::KwConst as u32 => TokenType::KwConst,
    t if t == TokenType::KwContinue as u32 => TokenType::KwContinue,
    t if t == TokenType::KwContractEnsures as u32 =>
        TokenType::KwContractEnsures,
    t if t == TokenType::KwContractRequires as u32 =>
        TokenType::KwContractRequires,
    t if t == TokenType::KwCrate as u32 => TokenType::KwCrate,
    t if t == TokenType::KwDefault as u32 => TokenType::KwDefault,
    t if t == TokenType::KwDyn as u32 => TokenType::KwDyn,
    t if t == TokenType::KwElse as u32 => TokenType::KwElse,
    t if t == TokenType::KwEnum as u32 => TokenType::KwEnum,
    t if t == TokenType::KwExtern as u32 => TokenType::KwExtern,
    t if t == TokenType::KwFn as u32 => TokenType::KwFn,
    t if t == TokenType::KwFor as u32 => TokenType::KwFor,
    t if t == TokenType::KwFinal as u32 => TokenType::KwFinal,
    t if t == TokenType::KwGen as u32 => TokenType::KwGen,
    t if t == TokenType::KwIf as u32 => TokenType::KwIf,
    t if t == TokenType::KwImpl as u32 => TokenType::KwImpl,
    t if t == TokenType::KwIn as u32 => TokenType::KwIn,
    t if t == TokenType::KwLet as u32 => TokenType::KwLet,
    t if t == TokenType::KwLoop as u32 => TokenType::KwLoop,
    t if t == TokenType::KwMacro as u32 => TokenType::KwMacro,
    t if t == TokenType::KwMacroRules as u32 => TokenType::KwMacroRules,
    t if t == TokenType::KwMatch as u32 => TokenType::KwMatch,
    t if t == TokenType::KwMod as u32 => TokenType::KwMod,
    t if t == TokenType::KwMove as u32 => TokenType::KwMove,
    t if t == TokenType::KwMut as u32 => TokenType::KwMut,
    t if t == TokenType::KwPub as u32 => TokenType::KwPub,
    t if t == TokenType::KwRaw as u32 => TokenType::KwRaw,
    t if t == TokenType::KwRef as u32 => TokenType::KwRef,
    t if t == TokenType::KwReturn as u32 => TokenType::KwReturn,
    t if t == TokenType::KwReuse as u32 => TokenType::KwReuse,
    t if t == TokenType::KwSafe as u32 => TokenType::KwSafe,
    t if t == TokenType::KwSelfUpper as u32 => TokenType::KwSelfUpper,
    t if t == TokenType::KwStatic as u32 => TokenType::KwStatic,
    t if t == TokenType::KwStruct as u32 => TokenType::KwStruct,
    t if t == TokenType::KwSuper as u32 => TokenType::KwSuper,
    t if t == TokenType::KwTrait as u32 => TokenType::KwTrait,
    t if t == TokenType::KwTry as u32 => TokenType::KwTry,
    t if t == TokenType::KwType as u32 => TokenType::KwType,
    t if t == TokenType::KwUnderscore as u32 => TokenType::KwUnderscore,
    t if t == TokenType::KwUnsafe as u32 => TokenType::KwUnsafe,
    t if t == TokenType::KwUse as u32 => TokenType::KwUse,
    t if t == TokenType::KwWhere as u32 => TokenType::KwWhere,
    t if t == TokenType::KwWhile as u32 => TokenType::KwWhile,
    t if t == TokenType::KwYield as u32 => TokenType::KwYield,
    t if t == TokenType::SymAttSyntax as u32 => TokenType::SymAttSyntax,
    t if t == TokenType::SymClobberAbi as u32 => TokenType::SymClobberAbi,
    t if t == TokenType::SymInlateout as u32 => TokenType::SymInlateout,
    t if t == TokenType::SymInout as u32 => TokenType::SymInout,
    t if t == TokenType::SymIs as u32 => TokenType::SymIs,
    t if t == TokenType::SymLabel as u32 => TokenType::SymLabel,
    t if t == TokenType::SymLateout as u32 => TokenType::SymLateout,
    t if t == TokenType::SymMayUnwind as u32 => TokenType::SymMayUnwind,
    t if t == TokenType::SymNomem as u32 => TokenType::SymNomem,
    t if t == TokenType::SymNoreturn as u32 => TokenType::SymNoreturn,
    t if t == TokenType::SymNostack as u32 => TokenType::SymNostack,
    t if t == TokenType::SymNull as u32 => TokenType::SymNull,
    t if t == TokenType::SymOptions as u32 => TokenType::SymOptions,
    t if t == TokenType::SymOut as u32 => TokenType::SymOut,
    t if t == TokenType::SymPreservesFlags as u32 =>
        TokenType::SymPreservesFlags,
    t if t == TokenType::SymPure as u32 => TokenType::SymPure,
    t if t == TokenType::SymReadonly as u32 => TokenType::SymReadonly,
    t if t == TokenType::SymSym as u32 => TokenType::SymSym,
    _ => {
        ::core::panicking::panic_fmt(format_args!("unhandled value: {0}",
                val));
    }
}from_u32_match! { val;
173            Eq,
174            Lt,
175            Le,
176            EqEq,
177            Gt,
178            AndAnd,
179            OrOr,
180            Bang,
181            Tilde,
182
183            Plus,
184            Minus,
185            Star,
186            And,
187            Or,
188
189            At,
190            Dot,
191            DotDot,
192            DotDotDot,
193            DotDotEq,
194            Comma,
195            Semi,
196            Colon,
197            PathSep,
198            RArrow,
199            FatArrow,
200            Pound,
201            Question,
202            OpenParen,
203            CloseParen,
204            OpenBrace,
205            CloseBrace,
206            OpenBracket,
207            CloseBracket,
208            Eof,
209
210            Operator,
211            Ident,
212            Lifetime,
213            Path,
214            Type,
215            Const,
216
217            KwAs,
218            KwAsync,
219            KwAuto,
220            KwAwait,
221            KwBecome,
222            KwBox,
223            KwBreak,
224            KwCatch,
225            KwConst,
226            KwContinue,
227            KwContractEnsures,
228            KwContractRequires,
229            KwCrate,
230            KwDefault,
231            KwDyn,
232            KwElse,
233            KwEnum,
234            KwExtern,
235            KwFn,
236            KwFor,
237            KwFinal,
238            KwGen,
239            KwIf,
240            KwImpl,
241            KwIn,
242            KwLet,
243            KwLoop,
244            KwMacro,
245            KwMacroRules,
246            KwMatch,
247            KwMod,
248            KwMove,
249            KwMut,
250            KwPub,
251            KwRaw,
252            KwRef,
253            KwReturn,
254            KwReuse,
255            KwSafe,
256            KwSelfUpper,
257            KwStatic,
258            KwStruct,
259            KwSuper,
260            KwTrait,
261            KwTry,
262            KwType,
263            KwUnderscore,
264            KwUnsafe,
265            KwUse,
266            KwWhere,
267            KwWhile,
268            KwYield,
269
270            SymAttSyntax,
271            SymClobberAbi,
272            SymInlateout,
273            SymInout,
274            SymIs,
275            SymLabel,
276            SymLateout,
277            SymMayUnwind,
278            SymNomem,
279            SymNoreturn,
280            SymNostack,
281            SymNull,
282            SymOptions,
283            SymOut,
284            SymPreservesFlags,
285            SymPure,
286            SymReadonly,
287            SymSym,
288        };
289        token_type
290    }
291
292    pub(super) fn is_keyword(&self) -> Option<Symbol> {
293        match self {
294            TokenType::KwAs => Some(kw::As),
295            TokenType::KwAsync => Some(kw::Async),
296            TokenType::KwAuto => Some(kw::Auto),
297            TokenType::KwAwait => Some(kw::Await),
298            TokenType::KwBecome => Some(kw::Become),
299            TokenType::KwBox => Some(kw::Box),
300            TokenType::KwBreak => Some(kw::Break),
301            TokenType::KwCatch => Some(kw::Catch),
302            TokenType::KwConst => Some(kw::Const),
303            TokenType::KwContinue => Some(kw::Continue),
304            TokenType::KwContractEnsures => Some(kw::ContractEnsures),
305            TokenType::KwContractRequires => Some(kw::ContractRequires),
306            TokenType::KwCrate => Some(kw::Crate),
307            TokenType::KwDefault => Some(kw::Default),
308            TokenType::KwDyn => Some(kw::Dyn),
309            TokenType::KwElse => Some(kw::Else),
310            TokenType::KwEnum => Some(kw::Enum),
311            TokenType::KwExtern => Some(kw::Extern),
312            TokenType::KwFn => Some(kw::Fn),
313            TokenType::KwFor => Some(kw::For),
314            TokenType::KwFinal => Some(kw::Final),
315            TokenType::KwGen => Some(kw::Gen),
316            TokenType::KwIf => Some(kw::If),
317            TokenType::KwImpl => Some(kw::Impl),
318            TokenType::KwIn => Some(kw::In),
319            TokenType::KwLet => Some(kw::Let),
320            TokenType::KwLoop => Some(kw::Loop),
321            TokenType::KwMacroRules => Some(kw::MacroRules),
322            TokenType::KwMacro => Some(kw::Macro),
323            TokenType::KwMatch => Some(kw::Match),
324            TokenType::KwMod => Some(kw::Mod),
325            TokenType::KwMove => Some(kw::Move),
326            TokenType::KwMut => Some(kw::Mut),
327            TokenType::KwPub => Some(kw::Pub),
328            TokenType::KwRaw => Some(kw::Raw),
329            TokenType::KwRef => Some(kw::Ref),
330            TokenType::KwReturn => Some(kw::Return),
331            TokenType::KwReuse => Some(kw::Reuse),
332            TokenType::KwSafe => Some(kw::Safe),
333            TokenType::KwSelfUpper => Some(kw::SelfUpper),
334            TokenType::KwStatic => Some(kw::Static),
335            TokenType::KwStruct => Some(kw::Struct),
336            TokenType::KwSuper => Some(kw::Super),
337            TokenType::KwTrait => Some(kw::Trait),
338            TokenType::KwTry => Some(kw::Try),
339            TokenType::KwType => Some(kw::Type),
340            TokenType::KwUnderscore => Some(kw::Underscore),
341            TokenType::KwUnsafe => Some(kw::Unsafe),
342            TokenType::KwUse => Some(kw::Use),
343            TokenType::KwWhere => Some(kw::Where),
344            TokenType::KwWhile => Some(kw::While),
345            TokenType::KwYield => Some(kw::Yield),
346
347            TokenType::SymAttSyntax => Some(sym::att_syntax),
348            TokenType::SymClobberAbi => Some(sym::clobber_abi),
349            TokenType::SymInlateout => Some(sym::inlateout),
350            TokenType::SymInout => Some(sym::inout),
351            TokenType::SymIs => Some(sym::is),
352            TokenType::SymLabel => Some(sym::label),
353            TokenType::SymLateout => Some(sym::lateout),
354            TokenType::SymMayUnwind => Some(sym::may_unwind),
355            TokenType::SymNomem => Some(sym::nomem),
356            TokenType::SymNoreturn => Some(sym::noreturn),
357            TokenType::SymNostack => Some(sym::nostack),
358            TokenType::SymNull => Some(sym::null),
359            TokenType::SymOptions => Some(sym::options),
360            TokenType::SymOut => Some(sym::out),
361            TokenType::SymPreservesFlags => Some(sym::preserves_flags),
362            TokenType::SymPure => Some(sym::pure),
363            TokenType::SymReadonly => Some(sym::readonly),
364            TokenType::SymSym => Some(sym::sym),
365            _ => None,
366        }
367    }
368
369    // The output should be the same as that produced by
370    // `rustc_ast_pretty::pprust::token_to_string`.
371    pub(super) fn to_string(&self) -> String {
372        match self {
373            TokenType::Eq => "`=`",
374            TokenType::Lt => "`<`",
375            TokenType::Le => "`<=`",
376            TokenType::EqEq => "`==`",
377            TokenType::Gt => "`>`",
378            TokenType::AndAnd => "`&&`",
379            TokenType::OrOr => "`||`",
380            TokenType::Bang => "`!`",
381            TokenType::Tilde => "`~`",
382
383            TokenType::Plus => "`+`",
384            TokenType::Minus => "`-`",
385            TokenType::Star => "`*`",
386            TokenType::And => "`&`",
387            TokenType::Or => "`|`",
388
389            TokenType::At => "`@`",
390            TokenType::Dot => "`.`",
391            TokenType::DotDot => "`..`",
392            TokenType::DotDotDot => "`...`",
393            TokenType::DotDotEq => "`..=`",
394            TokenType::Comma => "`,`",
395            TokenType::Semi => "`;`",
396            TokenType::Colon => "`:`",
397            TokenType::PathSep => "`::`",
398            TokenType::RArrow => "`->`",
399            TokenType::FatArrow => "`=>`",
400            TokenType::Pound => "`#`",
401            TokenType::Question => "`?`",
402            TokenType::OpenParen => "`(`",
403            TokenType::CloseParen => "`)`",
404            TokenType::OpenBrace => "`{`",
405            TokenType::CloseBrace => "`}`",
406            TokenType::OpenBracket => "`[`",
407            TokenType::CloseBracket => "`]`",
408            TokenType::Eof => "<eof>",
409
410            TokenType::Operator => "an operator",
411            TokenType::Ident => "identifier",
412            TokenType::Lifetime => "lifetime",
413            TokenType::Path => "path",
414            TokenType::Type => "type",
415            TokenType::Const => "a const expression",
416
417            _ => return ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("`{0}`",
                self.is_keyword().unwrap()))
    })format!("`{}`", self.is_keyword().unwrap()),
418        }
419        .to_string()
420    }
421}
422
423/// Used by various `Parser` methods such as `check` and `eat`. The first field
424/// is always by used those methods. The second field is only used when the
425/// first field doesn't match.
426#[derive(#[automatically_derived]
impl ::core::clone::Clone for ExpTokenPair {
    #[inline]
    fn clone(&self) -> ExpTokenPair {
        let _: ::core::clone::AssertParamIsClone<TokenKind>;
        let _: ::core::clone::AssertParamIsClone<TokenType>;
        *self
    }
}Clone, #[automatically_derived]
impl ::core::marker::Copy for ExpTokenPair { }Copy, #[automatically_derived]
impl ::core::fmt::Debug for ExpTokenPair {
    #[inline]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::debug_struct_field2_finish(f, "ExpTokenPair",
            "tok", &self.tok, "token_type", &&self.token_type)
    }
}Debug)]
427pub struct ExpTokenPair {
428    pub tok: TokenKind,
429    pub token_type: TokenType,
430}
431
432/// Used by various `Parser` methods such as `check_keyword` and `eat_keyword`.
433/// The first field is always used by those methods. The second field is only
434/// used when the first field doesn't match.
435#[derive(#[automatically_derived]
impl ::core::clone::Clone for ExpKeywordPair {
    #[inline]
    fn clone(&self) -> ExpKeywordPair {
        let _: ::core::clone::AssertParamIsClone<Symbol>;
        let _: ::core::clone::AssertParamIsClone<TokenType>;
        *self
    }
}Clone, #[automatically_derived]
impl ::core::marker::Copy for ExpKeywordPair { }Copy)]
436pub struct ExpKeywordPair {
437    pub kw: Symbol,
438    pub token_type: TokenType,
439}
440
441// Gets a statically-known `ExpTokenPair` pair (for non-keywords) or
442// `ExpKeywordPair` (for keywords), as used with various `check`/`expect`
443// methods in `Parser`.
444//
445// The name is short because it's used a lot.
446#[macro_export]
447// We don't use the normal `#[rustfmt::skip]` here because that triggers a
448// bogus "macro-expanded `macro_export` macros from the current crate cannot be
449// referred to by absolute paths" error, ugh. See #52234.
450#[cfg_attr(rustfmt, rustfmt::skip)]
451macro_rules! exp {
452    // `ExpTokenPair` helper rules.
453    (@tok, $tok:ident) => {
454        $crate::parser::token_type::ExpTokenPair {
455            tok: rustc_ast::token::$tok,
456            token_type: $crate::parser::token_type::TokenType::$tok
457        }
458    };
459
460    // `ExpKeywordPair` helper rules.
461    (@kw, $kw:ident, $token_type:ident) => {
462        $crate::parser::token_type::ExpKeywordPair {
463            kw: rustc_span::symbol::kw::$kw,
464            token_type: $crate::parser::token_type::TokenType::$token_type,
465        }
466    };
467    (@sym, $kw:ident, $token_type:ident) => {
468        $crate::parser::token_type::ExpKeywordPair {
469            kw: rustc_span::symbol::sym::$kw,
470            token_type: $crate::parser::token_type::TokenType::$token_type,
471        }
472    };
473
474    (Eq)             => { exp!(@tok, Eq) };
475    (Lt)             => { exp!(@tok, Lt) };
476    (Le)             => { exp!(@tok, Le) };
477    (EqEq)           => { exp!(@tok, EqEq) };
478    (Gt)             => { exp!(@tok, Gt) };
479    (AndAnd)         => { exp!(@tok, AndAnd) };
480    (OrOr)           => { exp!(@tok, OrOr) };
481    (Bang)           => { exp!(@tok, Bang) };
482    (Tilde)          => { exp!(@tok, Tilde) };
483    (Plus)           => { exp!(@tok, Plus) };
484    (Minus)          => { exp!(@tok, Minus) };
485    (Star)           => { exp!(@tok, Star) };
486    (And)            => { exp!(@tok, And) };
487    (Or)             => { exp!(@tok, Or) };
488    (At)             => { exp!(@tok, At) };
489    (Dot)            => { exp!(@tok, Dot) };
490    (DotDot)         => { exp!(@tok, DotDot) };
491    (DotDotDot)      => { exp!(@tok, DotDotDot) };
492    (DotDotEq)       => { exp!(@tok, DotDotEq) };
493    (Comma)          => { exp!(@tok, Comma) };
494    (Semi)           => { exp!(@tok, Semi) };
495    (Colon)          => { exp!(@tok, Colon) };
496    (PathSep)        => { exp!(@tok, PathSep) };
497    (RArrow)         => { exp!(@tok, RArrow) };
498    (FatArrow)       => { exp!(@tok, FatArrow) };
499    (Pound)          => { exp!(@tok, Pound) };
500    (Question)       => { exp!(@tok, Question) };
501    (Eof)            => { exp!(@tok, Eof) };
502
503    (OpenParen)      => { exp!(@tok, OpenParen) };
504    (OpenBrace)      => { exp!(@tok, OpenBrace) };
505    (OpenBracket)    => { exp!(@tok, OpenBracket) };
506    (CloseParen)     => { exp!(@tok, CloseParen) };
507    (CloseBrace)     => { exp!(@tok, CloseBrace) };
508    (CloseBracket)   => { exp!(@tok, CloseBracket) };
509
510    (As)             => { exp!(@kw, As,         KwAs) };
511    (Async)          => { exp!(@kw, Async,      KwAsync) };
512    (Auto)           => { exp!(@kw, Auto,       KwAuto) };
513    (Await)          => { exp!(@kw, Await,      KwAwait) };
514    (Become)         => { exp!(@kw, Become,     KwBecome) };
515    (Box)            => { exp!(@kw, Box,        KwBox) };
516    (Break)          => { exp!(@kw, Break,      KwBreak) };
517    (Catch)          => { exp!(@kw, Catch,      KwCatch) };
518    (Const)          => { exp!(@kw, Const,      KwConst) };
519    (Continue)       => { exp!(@kw, Continue,   KwContinue) };
520    (ContractEnsures)  => { exp!(@kw, ContractEnsures, KwContractEnsures) };
521    (ContractRequires) => { exp!(@kw, ContractRequires, KwContractRequires) };
522    (Crate)          => { exp!(@kw, Crate,      KwCrate) };
523    (Default)        => { exp!(@kw, Default,    KwDefault) };
524    (Dyn)            => { exp!(@kw, Dyn,        KwDyn) };
525    (Else)           => { exp!(@kw, Else,       KwElse) };
526    (Enum)           => { exp!(@kw, Enum,       KwEnum) };
527    (Extern)         => { exp!(@kw, Extern,     KwExtern) };
528    (Fn)             => { exp!(@kw, Fn,         KwFn) };
529    (For)            => { exp!(@kw, For,        KwFor) };
530    (Final)          => { exp!(@kw, Final,      KwFinal) };
531    (Gen)            => { exp!(@kw, Gen,        KwGen) };
532    (If)             => { exp!(@kw, If,         KwIf) };
533    (Impl)           => { exp!(@kw, Impl,       KwImpl) };
534    (In)             => { exp!(@kw, In,         KwIn) };
535    (Let)            => { exp!(@kw, Let,        KwLet) };
536    (Loop)           => { exp!(@kw, Loop,       KwLoop) };
537    (Macro)          => { exp!(@kw, Macro,      KwMacro) };
538    (MacroRules)     => { exp!(@kw, MacroRules, KwMacroRules) };
539    (Match)          => { exp!(@kw, Match,      KwMatch) };
540    (Mod)            => { exp!(@kw, Mod,        KwMod) };
541    (Move)           => { exp!(@kw, Move,       KwMove) };
542    (Mut)            => { exp!(@kw, Mut,        KwMut) };
543    (Pub)            => { exp!(@kw, Pub,        KwPub) };
544    (Raw)            => { exp!(@kw, Raw,        KwRaw) };
545    (Ref)            => { exp!(@kw, Ref,        KwRef) };
546    (Return)         => { exp!(@kw, Return,     KwReturn) };
547    (Reuse)          => { exp!(@kw, Reuse,      KwReuse) };
548    (Safe)           => { exp!(@kw, Safe,       KwSafe) };
549    (SelfUpper)      => { exp!(@kw, SelfUpper,  KwSelfUpper) };
550    (Static)         => { exp!(@kw, Static,     KwStatic) };
551    (Struct)         => { exp!(@kw, Struct,     KwStruct) };
552    (Super)          => { exp!(@kw, Super,      KwSuper) };
553    (Trait)          => { exp!(@kw, Trait,      KwTrait) };
554    (Try)            => { exp!(@kw, Try,        KwTry) };
555    (Type)           => { exp!(@kw, Type,       KwType) };
556    (Underscore)     => { exp!(@kw, Underscore, KwUnderscore) };
557    (Unsafe)         => { exp!(@kw, Unsafe,     KwUnsafe) };
558    (Use)            => { exp!(@kw, Use,        KwUse) };
559    (Where)          => { exp!(@kw, Where,      KwWhere) };
560    (While)          => { exp!(@kw, While,      KwWhile) };
561    (Yield)          => { exp!(@kw, Yield,      KwYield) };
562
563    (AttSyntax)      => { exp!(@sym, att_syntax,      SymAttSyntax) };
564    (Bikeshed)       => { exp!(@sym, bikeshed,        SymBikeshed) };
565    (ClobberAbi)     => { exp!(@sym, clobber_abi,     SymClobberAbi) };
566    (Inlateout)      => { exp!(@sym, inlateout,       SymInlateout) };
567    (Inout)          => { exp!(@sym, inout,           SymInout) };
568    (Is)             => { exp!(@sym, is,              SymIs) };
569    (Label)          => { exp!(@sym, label,           SymLabel) };
570    (Lateout)        => { exp!(@sym, lateout,         SymLateout) };
571    (MayUnwind)      => { exp!(@sym, may_unwind,      SymMayUnwind) };
572    (Nomem)          => { exp!(@sym, nomem,           SymNomem) };
573    (Noreturn)       => { exp!(@sym, noreturn,        SymNoreturn) };
574    (Nostack)        => { exp!(@sym, nostack,         SymNostack) };
575    (Null)           => { exp!(@sym, null,            SymNull) };
576    (Options)        => { exp!(@sym, options,         SymOptions) };
577    (Out)            => { exp!(@sym, out,             SymOut) };
578    (Pin)            => { exp!(@sym, pin,             SymPin) };
579    (PreservesFlags) => { exp!(@sym, preserves_flags, SymPreservesFlags) };
580    (Pure)           => { exp!(@sym, pure,            SymPure) };
581    (Readonly)       => { exp!(@sym, readonly,        SymReadonly) };
582    (Sym)            => { exp!(@sym, sym,             SymSym) };
583}
584
585/// A bitset type designed specifically for `Parser::expected_token_types`,
586/// which is very hot. `u128` is the smallest integer that will fit every
587/// `TokenType` value.
588#[derive(#[automatically_derived]
impl ::core::clone::Clone for TokenTypeSet {
    #[inline]
    fn clone(&self) -> TokenTypeSet {
        let _: ::core::clone::AssertParamIsClone<u128>;
        *self
    }
}Clone, #[automatically_derived]
impl ::core::marker::Copy for TokenTypeSet { }Copy)]
589pub(super) struct TokenTypeSet(u128);
590
591impl TokenTypeSet {
592    pub(super) const fn new() -> TokenTypeSet {
593        TokenTypeSet(0)
594    }
595
596    pub(super) fn is_empty(&self) -> bool {
597        self.0 == 0
598    }
599
600    pub(super) fn insert(&mut self, token_type: TokenType) {
601        self.0 = self.0 | (1u128 << token_type as u32)
602    }
603
604    pub(super) fn clear(&mut self) {
605        self.0 = 0
606    }
607
608    pub(super) fn contains(&self, token_type: TokenType) -> bool {
609        self.0 & (1u128 << token_type as u32) != 0
610    }
611
612    pub(super) fn iter(&self) -> TokenTypeSetIter {
613        TokenTypeSetIter(*self)
614    }
615}
616
617// The `TokenTypeSet` is a copy of the set being iterated. It initially holds
618// the entire set. Each bit is cleared as it is returned. We have finished once
619// it is all zeroes.
620pub(super) struct TokenTypeSetIter(TokenTypeSet);
621
622impl Iterator for TokenTypeSetIter {
623    type Item = TokenType;
624
625    fn next(&mut self) -> Option<TokenType> {
626        let num_bits: u32 = (size_of_val(&self.0.0) * 8) as u32;
627        match (&num_bits, &128) {
    (left_val, right_val) => {
        if !(*left_val == *right_val) {
            let kind = ::core::panicking::AssertKind::Eq;
            ::core::panicking::assert_failed(kind, &*left_val, &*right_val,
                ::core::option::Option::None);
        }
    }
};assert_eq!(num_bits, 128);
628        let z = self.0.0.trailing_zeros();
629        if z == num_bits {
630            None
631        } else {
632            self.0.0 &= !(1 << z); // clear the trailing 1 bit
633            Some(TokenType::from_u32(z))
634        }
635    }
636}
rustc_parse/parser/token_type.rs

rustc_parse/parser/
token_type.rs
