rustc_parse/parser/
token_type.rs

1use rustc_ast::token::TokenKind;
2use rustc_span::symbol::{Symbol, kw, sym};
3
4/// Used in "expected"/"expected one of" error messages. Tokens are added here
5/// as necessary. Tokens with values (e.g. literals, identifiers) are
6/// represented by a single variant (e.g. `Literal`, `Ident`).
7///
8/// It's an awkward representation, but it's important for performance. It's a
9/// C-style parameterless enum so that `TokenTypeSet` can be a bitset. This is
10/// important because `Parser::expected_token_types` is very hot. `TokenType`
11/// used to have variants with parameters (e.g. all the keywords were in a
12/// single `Keyword` variant with a `Symbol` parameter) and
13/// `Parser::expected_token_types` was a `Vec<TokenType>` which was much slower
14/// to manipulate.
15///
16/// We really want to keep the number of variants to 128 or fewer, so that
17/// `TokenTypeSet` can be implemented with a `u128`.
18#[derive(#[automatically_derived]
impl ::core::fmt::Debug for TokenType {
    #[inline]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::write_str(f,
            match self {
                TokenType::Eq => "Eq",
                TokenType::Lt => "Lt",
                TokenType::Le => "Le",
                TokenType::EqEq => "EqEq",
                TokenType::Gt => "Gt",
                TokenType::AndAnd => "AndAnd",
                TokenType::OrOr => "OrOr",
                TokenType::Bang => "Bang",
                TokenType::Tilde => "Tilde",
                TokenType::Plus => "Plus",
                TokenType::Minus => "Minus",
                TokenType::Star => "Star",
                TokenType::And => "And",
                TokenType::Or => "Or",
                TokenType::At => "At",
                TokenType::Dot => "Dot",
                TokenType::DotDot => "DotDot",
                TokenType::DotDotDot => "DotDotDot",
                TokenType::DotDotEq => "DotDotEq",
                TokenType::Comma => "Comma",
                TokenType::Semi => "Semi",
                TokenType::Colon => "Colon",
                TokenType::PathSep => "PathSep",
                TokenType::RArrow => "RArrow",
                TokenType::FatArrow => "FatArrow",
                TokenType::Pound => "Pound",
                TokenType::Question => "Question",
                TokenType::OpenParen => "OpenParen",
                TokenType::CloseParen => "CloseParen",
                TokenType::OpenBrace => "OpenBrace",
                TokenType::CloseBrace => "CloseBrace",
                TokenType::OpenBracket => "OpenBracket",
                TokenType::CloseBracket => "CloseBracket",
                TokenType::Eof => "Eof",
                TokenType::Operator => "Operator",
                TokenType::Ident => "Ident",
                TokenType::Lifetime => "Lifetime",
                TokenType::Path => "Path",
                TokenType::Type => "Type",
                TokenType::Const => "Const",
                TokenType::KwAs => "KwAs",
                TokenType::KwAsync => "KwAsync",
                TokenType::KwAuto => "KwAuto",
                TokenType::KwAwait => "KwAwait",
                TokenType::KwBecome => "KwBecome",
                TokenType::KwBox => "KwBox",
                TokenType::KwBreak => "KwBreak",
                TokenType::KwCatch => "KwCatch",
                TokenType::KwConst => "KwConst",
                TokenType::KwContinue => "KwContinue",
                TokenType::KwContractEnsures => "KwContractEnsures",
                TokenType::KwContractRequires => "KwContractRequires",
                TokenType::KwCrate => "KwCrate",
                TokenType::KwDefault => "KwDefault",
                TokenType::KwDyn => "KwDyn",
                TokenType::KwElse => "KwElse",
                TokenType::KwEnum => "KwEnum",
                TokenType::KwExtern => "KwExtern",
                TokenType::KwFinal => "KwFinal",
                TokenType::KwFn => "KwFn",
                TokenType::KwFor => "KwFor",
                TokenType::KwGen => "KwGen",
                TokenType::KwIf => "KwIf",
                TokenType::KwImpl => "KwImpl",
                TokenType::KwIn => "KwIn",
                TokenType::KwLet => "KwLet",
                TokenType::KwLoop => "KwLoop",
                TokenType::KwMacro => "KwMacro",
                TokenType::KwMacroRules => "KwMacroRules",
                TokenType::KwMatch => "KwMatch",
                TokenType::KwMod => "KwMod",
                TokenType::KwMove => "KwMove",
                TokenType::KwMut => "KwMut",
                TokenType::KwPin => "KwPin",
                TokenType::KwPub => "KwPub",
                TokenType::KwRaw => "KwRaw",
                TokenType::KwRef => "KwRef",
                TokenType::KwReturn => "KwReturn",
                TokenType::KwReuse => "KwReuse",
                TokenType::KwSafe => "KwSafe",
                TokenType::KwSelfUpper => "KwSelfUpper",
                TokenType::KwStatic => "KwStatic",
                TokenType::KwStruct => "KwStruct",
                TokenType::KwSuper => "KwSuper",
                TokenType::KwTrait => "KwTrait",
                TokenType::KwTry => "KwTry",
                TokenType::KwType => "KwType",
                TokenType::KwUnderscore => "KwUnderscore",
                TokenType::KwUnsafe => "KwUnsafe",
                TokenType::KwUse => "KwUse",
                TokenType::KwWhere => "KwWhere",
                TokenType::KwWhile => "KwWhile",
                TokenType::KwYield => "KwYield",
                TokenType::SymAttSyntax => "SymAttSyntax",
                TokenType::SymBikeshed => "SymBikeshed",
                TokenType::SymClobberAbi => "SymClobberAbi",
                TokenType::SymInlateout => "SymInlateout",
                TokenType::SymInout => "SymInout",
                TokenType::SymIs => "SymIs",
                TokenType::SymLabel => "SymLabel",
                TokenType::SymLateout => "SymLateout",
                TokenType::SymMayUnwind => "SymMayUnwind",
                TokenType::SymNomem => "SymNomem",
                TokenType::SymNoreturn => "SymNoreturn",
                TokenType::SymNostack => "SymNostack",
                TokenType::SymNull => "SymNull",
                TokenType::SymOptions => "SymOptions",
                TokenType::SymOut => "SymOut",
                TokenType::SymPin => "SymPin",
                TokenType::SymPreservesFlags => "SymPreservesFlags",
                TokenType::SymPure => "SymPure",
                TokenType::SymReadonly => "SymReadonly",
                TokenType::SymSym => "SymSym",
            })
    }
}Debug, #[automatically_derived]
#[doc(hidden)]
unsafe impl ::core::clone::TrivialClone for TokenType { }
#[automatically_derived]
impl ::core::clone::Clone for TokenType {
    #[inline]
    fn clone(&self) -> TokenType { *self }
}Clone, #[automatically_derived]
impl ::core::marker::Copy for TokenType { }Copy, #[automatically_derived]
impl ::core::marker::StructuralPartialEq for TokenType { }
#[automatically_derived]
impl ::core::cmp::PartialEq for TokenType {
    #[inline]
    fn eq(&self, other: &TokenType) -> bool {
        let __self_discr = ::core::intrinsics::discriminant_value(self);
        let __arg1_discr = ::core::intrinsics::discriminant_value(other);
        __self_discr == __arg1_discr
    }
}PartialEq)]
19pub enum TokenType {
20    // Expression-operator symbols
21    Eq,
22    Lt,
23    Le,
24    EqEq,
25    Gt,
26    AndAnd,
27    OrOr,
28    Bang,
29    Tilde,
30
31    // BinOps
32    Plus,
33    Minus,
34    Star,
35    And,
36    Or,
37
38    // Structural symbols
39    At,
40    Dot,
41    DotDot,
42    DotDotDot,
43    DotDotEq,
44    Comma,
45    Semi,
46    Colon,
47    PathSep,
48    RArrow,
49    FatArrow,
50    Pound,
51    Question,
52    OpenParen,
53    CloseParen,
54    OpenBrace,
55    CloseBrace,
56    OpenBracket,
57    CloseBracket,
58    Eof,
59
60    // Token types with some details elided.
61    /// Any operator.
62    Operator,
63    /// Any identifier token.
64    Ident,
65    /// Any lifetime token.
66    Lifetime,
67    /// Any token that can start a path.
68    Path,
69    /// Any token that can start a type.
70    Type,
71    /// Any token that can start a const expression.
72    Const,
73
74    // Keywords
75    // tidy-alphabetical-start
76    KwAs,
77    KwAsync,
78    KwAuto,
79    KwAwait,
80    KwBecome,
81    KwBox,
82    KwBreak,
83    KwCatch,
84    KwConst,
85    KwContinue,
86    KwContractEnsures,
87    KwContractRequires,
88    KwCrate,
89    KwDefault,
90    KwDyn,
91    KwElse,
92    KwEnum,
93    KwExtern,
94    KwFinal,
95    KwFn,
96    KwFor,
97    KwGen,
98    KwIf,
99    KwImpl,
100    KwIn,
101    KwLet,
102    KwLoop,
103    KwMacro,
104    KwMacroRules,
105    KwMatch,
106    KwMod,
107    KwMove,
108    KwMut,
109    KwPin,
110    KwPub,
111    KwRaw,
112    KwRef,
113    KwReturn,
114    KwReuse,
115    KwSafe,
116    KwSelfUpper,
117    KwStatic,
118    KwStruct,
119    KwSuper,
120    KwTrait,
121    KwTry,
122    KwType,
123    KwUnderscore,
124    KwUnsafe,
125    KwUse,
126    KwWhere,
127    KwWhile,
128    KwYield,
129    // tidy-alphabetical-end
130
131    // Keyword-like symbols.
132    // tidy-alphabetical-start
133    SymAttSyntax,
134    SymBikeshed,
135    SymClobberAbi,
136    SymInlateout,
137    SymInout,
138    SymIs,
139    SymLabel,
140    SymLateout,
141    SymMayUnwind,
142    SymNomem,
143    SymNoreturn,
144    SymNostack,
145    SymNull,
146    SymOptions,
147    SymOut,
148    SymPin,
149    SymPreservesFlags,
150    SymPure,
151    SymReadonly,
152    SymSym,
153    // tidy-alphabetical-end
154}
155
156// Macro to avoid repetitive boilerplate code.
157macro_rules! from_u32_match {
158    ($val:ident; $($tok:ident,)+) => {
159        // A more obvious formulation would be `0 => TokenType::Eq`. But
160        // this formulation with the guard lets us avoid specifying a
161        // specific integer for each variant.
162        match $val {
163            $(
164                t if t == TokenType::$tok as u32 => TokenType::$tok,
165            )+
166            _ => panic!("unhandled value: {}", $val),
167        }
168    };
169}
170
171impl TokenType {
172    fn from_u32(val: u32) -> TokenType {
173        let token_type = match val {
    t if t == TokenType::Eq as u32 => TokenType::Eq,
    t if t == TokenType::Lt as u32 => TokenType::Lt,
    t if t == TokenType::Le as u32 => TokenType::Le,
    t if t == TokenType::EqEq as u32 => TokenType::EqEq,
    t if t == TokenType::Gt as u32 => TokenType::Gt,
    t if t == TokenType::AndAnd as u32 => TokenType::AndAnd,
    t if t == TokenType::OrOr as u32 => TokenType::OrOr,
    t if t == TokenType::Bang as u32 => TokenType::Bang,
    t if t == TokenType::Tilde as u32 => TokenType::Tilde,
    t if t == TokenType::Plus as u32 => TokenType::Plus,
    t if t == TokenType::Minus as u32 => TokenType::Minus,
    t if t == TokenType::Star as u32 => TokenType::Star,
    t if t == TokenType::And as u32 => TokenType::And,
    t if t == TokenType::Or as u32 => TokenType::Or,
    t if t == TokenType::At as u32 => TokenType::At,
    t if t == TokenType::Dot as u32 => TokenType::Dot,
    t if t == TokenType::DotDot as u32 => TokenType::DotDot,
    t if t == TokenType::DotDotDot as u32 => TokenType::DotDotDot,
    t if t == TokenType::DotDotEq as u32 => TokenType::DotDotEq,
    t if t == TokenType::Comma as u32 => TokenType::Comma,
    t if t == TokenType::Semi as u32 => TokenType::Semi,
    t if t == TokenType::Colon as u32 => TokenType::Colon,
    t if t == TokenType::PathSep as u32 => TokenType::PathSep,
    t if t == TokenType::RArrow as u32 => TokenType::RArrow,
    t if t == TokenType::FatArrow as u32 => TokenType::FatArrow,
    t if t == TokenType::Pound as u32 => TokenType::Pound,
    t if t == TokenType::Question as u32 => TokenType::Question,
    t if t == TokenType::OpenParen as u32 => TokenType::OpenParen,
    t if t == TokenType::CloseParen as u32 => TokenType::CloseParen,
    t if t == TokenType::OpenBrace as u32 => TokenType::OpenBrace,
    t if t == TokenType::CloseBrace as u32 => TokenType::CloseBrace,
    t if t == TokenType::OpenBracket as u32 => TokenType::OpenBracket,
    t if t == TokenType::CloseBracket as u32 => TokenType::CloseBracket,
    t if t == TokenType::Eof as u32 => TokenType::Eof,
    t if t == TokenType::Operator as u32 => TokenType::Operator,
    t if t == TokenType::Ident as u32 => TokenType::Ident,
    t if t == TokenType::Lifetime as u32 => TokenType::Lifetime,
    t if t == TokenType::Path as u32 => TokenType::Path,
    t if t == TokenType::Type as u32 => TokenType::Type,
    t if t == TokenType::Const as u32 => TokenType::Const,
    t if t == TokenType::KwAs as u32 => TokenType::KwAs,
    t if t == TokenType::KwAsync as u32 => TokenType::KwAsync,
    t if t == TokenType::KwAuto as u32 => TokenType::KwAuto,
    t if t == TokenType::KwAwait as u32 => TokenType::KwAwait,
    t if t == TokenType::KwBecome as u32 => TokenType::KwBecome,
    t if t == TokenType::KwBox as u32 => TokenType::KwBox,
    t if t == TokenType::KwBreak as u32 => TokenType::KwBreak,
    t if t == TokenType::KwCatch as u32 => TokenType::KwCatch,
    t if t == TokenType::KwConst as u32 => TokenType::KwConst,
    t if t == TokenType::KwContinue as u32 => TokenType::KwContinue,
    t if t == TokenType::KwContractEnsures as u32 =>
        TokenType::KwContractEnsures,
    t if t == TokenType::KwContractRequires as u32 =>
        TokenType::KwContractRequires,
    t if t == TokenType::KwCrate as u32 => TokenType::KwCrate,
    t if t == TokenType::KwDefault as u32 => TokenType::KwDefault,
    t if t == TokenType::KwDyn as u32 => TokenType::KwDyn,
    t if t == TokenType::KwElse as u32 => TokenType::KwElse,
    t if t == TokenType::KwEnum as u32 => TokenType::KwEnum,
    t if t == TokenType::KwExtern as u32 => TokenType::KwExtern,
    t if t == TokenType::KwFn as u32 => TokenType::KwFn,
    t if t == TokenType::KwFor as u32 => TokenType::KwFor,
    t if t == TokenType::KwFinal as u32 => TokenType::KwFinal,
    t if t == TokenType::KwGen as u32 => TokenType::KwGen,
    t if t == TokenType::KwIf as u32 => TokenType::KwIf,
    t if t == TokenType::KwImpl as u32 => TokenType::KwImpl,
    t if t == TokenType::KwIn as u32 => TokenType::KwIn,
    t if t == TokenType::KwLet as u32 => TokenType::KwLet,
    t if t == TokenType::KwLoop as u32 => TokenType::KwLoop,
    t if t == TokenType::KwMacro as u32 => TokenType::KwMacro,
    t if t == TokenType::KwMacroRules as u32 => TokenType::KwMacroRules,
    t if t == TokenType::KwMatch as u32 => TokenType::KwMatch,
    t if t == TokenType::KwMod as u32 => TokenType::KwMod,
    t if t == TokenType::KwMove as u32 => TokenType::KwMove,
    t if t == TokenType::KwMut as u32 => TokenType::KwMut,
    t if t == TokenType::KwPin as u32 => TokenType::KwPin,
    t if t == TokenType::KwPub as u32 => TokenType::KwPub,
    t if t == TokenType::KwRaw as u32 => TokenType::KwRaw,
    t if t == TokenType::KwRef as u32 => TokenType::KwRef,
    t if t == TokenType::KwReturn as u32 => TokenType::KwReturn,
    t if t == TokenType::KwReuse as u32 => TokenType::KwReuse,
    t if t == TokenType::KwSafe as u32 => TokenType::KwSafe,
    t if t == TokenType::KwSelfUpper as u32 => TokenType::KwSelfUpper,
    t if t == TokenType::KwStatic as u32 => TokenType::KwStatic,
    t if t == TokenType::KwStruct as u32 => TokenType::KwStruct,
    t if t == TokenType::KwSuper as u32 => TokenType::KwSuper,
    t if t == TokenType::KwTrait as u32 => TokenType::KwTrait,
    t if t == TokenType::KwTry as u32 => TokenType::KwTry,
    t if t == TokenType::KwType as u32 => TokenType::KwType,
    t if t == TokenType::KwUnderscore as u32 => TokenType::KwUnderscore,
    t if t == TokenType::KwUnsafe as u32 => TokenType::KwUnsafe,
    t if t == TokenType::KwUse as u32 => TokenType::KwUse,
    t if t == TokenType::KwWhere as u32 => TokenType::KwWhere,
    t if t == TokenType::KwWhile as u32 => TokenType::KwWhile,
    t if t == TokenType::KwYield as u32 => TokenType::KwYield,
    t if t == TokenType::SymAttSyntax as u32 => TokenType::SymAttSyntax,
    t if t == TokenType::SymClobberAbi as u32 => TokenType::SymClobberAbi,
    t if t == TokenType::SymInlateout as u32 => TokenType::SymInlateout,
    t if t == TokenType::SymInout as u32 => TokenType::SymInout,
    t if t == TokenType::SymIs as u32 => TokenType::SymIs,
    t if t == TokenType::SymLabel as u32 => TokenType::SymLabel,
    t if t == TokenType::SymLateout as u32 => TokenType::SymLateout,
    t if t == TokenType::SymMayUnwind as u32 => TokenType::SymMayUnwind,
    t if t == TokenType::SymNomem as u32 => TokenType::SymNomem,
    t if t == TokenType::SymNoreturn as u32 => TokenType::SymNoreturn,
    t if t == TokenType::SymNostack as u32 => TokenType::SymNostack,
    t if t == TokenType::SymNull as u32 => TokenType::SymNull,
    t if t == TokenType::SymOptions as u32 => TokenType::SymOptions,
    t if t == TokenType::SymOut as u32 => TokenType::SymOut,
    t if t == TokenType::SymPreservesFlags as u32 =>
        TokenType::SymPreservesFlags,
    t if t == TokenType::SymPure as u32 => TokenType::SymPure,
    t if t == TokenType::SymReadonly as u32 => TokenType::SymReadonly,
    t if t == TokenType::SymSym as u32 => TokenType::SymSym,
    _ => {
        ::core::panicking::panic_fmt(format_args!("unhandled value: {0}",
                val));
    }
}from_u32_match! { val;
174            Eq,
175            Lt,
176            Le,
177            EqEq,
178            Gt,
179            AndAnd,
180            OrOr,
181            Bang,
182            Tilde,
183
184            Plus,
185            Minus,
186            Star,
187            And,
188            Or,
189
190            At,
191            Dot,
192            DotDot,
193            DotDotDot,
194            DotDotEq,
195            Comma,
196            Semi,
197            Colon,
198            PathSep,
199            RArrow,
200            FatArrow,
201            Pound,
202            Question,
203            OpenParen,
204            CloseParen,
205            OpenBrace,
206            CloseBrace,
207            OpenBracket,
208            CloseBracket,
209            Eof,
210
211            Operator,
212            Ident,
213            Lifetime,
214            Path,
215            Type,
216            Const,
217
218            KwAs,
219            KwAsync,
220            KwAuto,
221            KwAwait,
222            KwBecome,
223            KwBox,
224            KwBreak,
225            KwCatch,
226            KwConst,
227            KwContinue,
228            KwContractEnsures,
229            KwContractRequires,
230            KwCrate,
231            KwDefault,
232            KwDyn,
233            KwElse,
234            KwEnum,
235            KwExtern,
236            KwFn,
237            KwFor,
238            KwFinal,
239            KwGen,
240            KwIf,
241            KwImpl,
242            KwIn,
243            KwLet,
244            KwLoop,
245            KwMacro,
246            KwMacroRules,
247            KwMatch,
248            KwMod,
249            KwMove,
250            KwMut,
251            KwPin,
252            KwPub,
253            KwRaw,
254            KwRef,
255            KwReturn,
256            KwReuse,
257            KwSafe,
258            KwSelfUpper,
259            KwStatic,
260            KwStruct,
261            KwSuper,
262            KwTrait,
263            KwTry,
264            KwType,
265            KwUnderscore,
266            KwUnsafe,
267            KwUse,
268            KwWhere,
269            KwWhile,
270            KwYield,
271
272            SymAttSyntax,
273            SymClobberAbi,
274            SymInlateout,
275            SymInout,
276            SymIs,
277            SymLabel,
278            SymLateout,
279            SymMayUnwind,
280            SymNomem,
281            SymNoreturn,
282            SymNostack,
283            SymNull,
284            SymOptions,
285            SymOut,
286            SymPreservesFlags,
287            SymPure,
288            SymReadonly,
289            SymSym,
290        };
291        token_type
292    }
293
294    pub(super) fn is_keyword(&self) -> Option<Symbol> {
295        match self {
296            TokenType::KwAs => Some(kw::As),
297            TokenType::KwAsync => Some(kw::Async),
298            TokenType::KwAuto => Some(kw::Auto),
299            TokenType::KwAwait => Some(kw::Await),
300            TokenType::KwBecome => Some(kw::Become),
301            TokenType::KwBox => Some(kw::Box),
302            TokenType::KwBreak => Some(kw::Break),
303            TokenType::KwCatch => Some(kw::Catch),
304            TokenType::KwConst => Some(kw::Const),
305            TokenType::KwContinue => Some(kw::Continue),
306            TokenType::KwContractEnsures => Some(kw::ContractEnsures),
307            TokenType::KwContractRequires => Some(kw::ContractRequires),
308            TokenType::KwCrate => Some(kw::Crate),
309            TokenType::KwDefault => Some(kw::Default),
310            TokenType::KwDyn => Some(kw::Dyn),
311            TokenType::KwElse => Some(kw::Else),
312            TokenType::KwEnum => Some(kw::Enum),
313            TokenType::KwExtern => Some(kw::Extern),
314            TokenType::KwFn => Some(kw::Fn),
315            TokenType::KwFor => Some(kw::For),
316            TokenType::KwFinal => Some(kw::Final),
317            TokenType::KwGen => Some(kw::Gen),
318            TokenType::KwIf => Some(kw::If),
319            TokenType::KwImpl => Some(kw::Impl),
320            TokenType::KwIn => Some(kw::In),
321            TokenType::KwLet => Some(kw::Let),
322            TokenType::KwLoop => Some(kw::Loop),
323            TokenType::KwMacroRules => Some(kw::MacroRules),
324            TokenType::KwMacro => Some(kw::Macro),
325            TokenType::KwMatch => Some(kw::Match),
326            TokenType::KwMod => Some(kw::Mod),
327            TokenType::KwMove => Some(kw::Move),
328            TokenType::KwMut => Some(kw::Mut),
329            TokenType::KwPin => Some(kw::Pin),
330            TokenType::KwPub => Some(kw::Pub),
331            TokenType::KwRaw => Some(kw::Raw),
332            TokenType::KwRef => Some(kw::Ref),
333            TokenType::KwReturn => Some(kw::Return),
334            TokenType::KwReuse => Some(kw::Reuse),
335            TokenType::KwSafe => Some(kw::Safe),
336            TokenType::KwSelfUpper => Some(kw::SelfUpper),
337            TokenType::KwStatic => Some(kw::Static),
338            TokenType::KwStruct => Some(kw::Struct),
339            TokenType::KwSuper => Some(kw::Super),
340            TokenType::KwTrait => Some(kw::Trait),
341            TokenType::KwTry => Some(kw::Try),
342            TokenType::KwType => Some(kw::Type),
343            TokenType::KwUnderscore => Some(kw::Underscore),
344            TokenType::KwUnsafe => Some(kw::Unsafe),
345            TokenType::KwUse => Some(kw::Use),
346            TokenType::KwWhere => Some(kw::Where),
347            TokenType::KwWhile => Some(kw::While),
348            TokenType::KwYield => Some(kw::Yield),
349
350            TokenType::SymAttSyntax => Some(sym::att_syntax),
351            TokenType::SymClobberAbi => Some(sym::clobber_abi),
352            TokenType::SymInlateout => Some(sym::inlateout),
353            TokenType::SymInout => Some(sym::inout),
354            TokenType::SymIs => Some(sym::is),
355            TokenType::SymLabel => Some(sym::label),
356            TokenType::SymLateout => Some(sym::lateout),
357            TokenType::SymMayUnwind => Some(sym::may_unwind),
358            TokenType::SymNomem => Some(sym::nomem),
359            TokenType::SymNoreturn => Some(sym::noreturn),
360            TokenType::SymNostack => Some(sym::nostack),
361            TokenType::SymNull => Some(sym::null),
362            TokenType::SymOptions => Some(sym::options),
363            TokenType::SymOut => Some(sym::out),
364            TokenType::SymPreservesFlags => Some(sym::preserves_flags),
365            TokenType::SymPure => Some(sym::pure),
366            TokenType::SymReadonly => Some(sym::readonly),
367            TokenType::SymSym => Some(sym::sym),
368            _ => None,
369        }
370    }
371
372    // The output should be the same as that produced by
373    // `rustc_ast_pretty::pprust::token_to_string`.
374    pub(super) fn to_string(&self) -> String {
375        match self {
376            TokenType::Eq => "`=`",
377            TokenType::Lt => "`<`",
378            TokenType::Le => "`<=`",
379            TokenType::EqEq => "`==`",
380            TokenType::Gt => "`>`",
381            TokenType::AndAnd => "`&&`",
382            TokenType::OrOr => "`||`",
383            TokenType::Bang => "`!`",
384            TokenType::Tilde => "`~`",
385
386            TokenType::Plus => "`+`",
387            TokenType::Minus => "`-`",
388            TokenType::Star => "`*`",
389            TokenType::And => "`&`",
390            TokenType::Or => "`|`",
391
392            TokenType::At => "`@`",
393            TokenType::Dot => "`.`",
394            TokenType::DotDot => "`..`",
395            TokenType::DotDotDot => "`...`",
396            TokenType::DotDotEq => "`..=`",
397            TokenType::Comma => "`,`",
398            TokenType::Semi => "`;`",
399            TokenType::Colon => "`:`",
400            TokenType::PathSep => "`::`",
401            TokenType::RArrow => "`->`",
402            TokenType::FatArrow => "`=>`",
403            TokenType::Pound => "`#`",
404            TokenType::Question => "`?`",
405            TokenType::OpenParen => "`(`",
406            TokenType::CloseParen => "`)`",
407            TokenType::OpenBrace => "`{`",
408            TokenType::CloseBrace => "`}`",
409            TokenType::OpenBracket => "`[`",
410            TokenType::CloseBracket => "`]`",
411            TokenType::Eof => "<eof>",
412
413            TokenType::Operator => "an operator",
414            TokenType::Ident => "identifier",
415            TokenType::Lifetime => "lifetime",
416            TokenType::Path => "path",
417            TokenType::Type => "type",
418            TokenType::Const => "a const expression",
419
420            _ => return ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("`{0}`",
                self.is_keyword().unwrap()))
    })format!("`{}`", self.is_keyword().unwrap()),
421        }
422        .to_string()
423    }
424}
425
426/// Used by various `Parser` methods such as `check` and `eat`. The first field
427/// is always by used those methods. The second field is only used when the
428/// first field doesn't match.
429#[derive(#[automatically_derived]
#[doc(hidden)]
unsafe impl ::core::clone::TrivialClone for ExpTokenPair { }
#[automatically_derived]
impl ::core::clone::Clone for ExpTokenPair {
    #[inline]
    fn clone(&self) -> ExpTokenPair {
        let _: ::core::clone::AssertParamIsClone<TokenKind>;
        let _: ::core::clone::AssertParamIsClone<TokenType>;
        *self
    }
}Clone, #[automatically_derived]
impl ::core::marker::Copy for ExpTokenPair { }Copy, #[automatically_derived]
impl ::core::fmt::Debug for ExpTokenPair {
    #[inline]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::debug_struct_field2_finish(f, "ExpTokenPair",
            "tok", &self.tok, "token_type", &&self.token_type)
    }
}Debug)]
430pub struct ExpTokenPair {
431    pub tok: TokenKind,
432    pub token_type: TokenType,
433}
434
435/// Used by various `Parser` methods such as `check_keyword` and `eat_keyword`.
436/// The first field is always used by those methods. The second field is only
437/// used when the first field doesn't match.
438#[derive(#[automatically_derived]
#[doc(hidden)]
unsafe impl ::core::clone::TrivialClone for ExpKeywordPair { }
#[automatically_derived]
impl ::core::clone::Clone for ExpKeywordPair {
    #[inline]
    fn clone(&self) -> ExpKeywordPair {
        let _: ::core::clone::AssertParamIsClone<Symbol>;
        let _: ::core::clone::AssertParamIsClone<TokenType>;
        *self
    }
}Clone, #[automatically_derived]
impl ::core::marker::Copy for ExpKeywordPair { }Copy)]
439pub struct ExpKeywordPair {
440    pub kw: Symbol,
441    pub token_type: TokenType,
442}
443
444// Gets a statically-known `ExpTokenPair` pair (for non-keywords) or
445// `ExpKeywordPair` (for keywords), as used with various `check`/`expect`
446// methods in `Parser`.
447//
448// The name is short because it's used a lot.
449#[macro_export]
450// We don't use the normal `#[rustfmt::skip]` here because that triggers a
451// bogus "macro-expanded `macro_export` macros from the current crate cannot be
452// referred to by absolute paths" error, ugh. See #52234.
453#[cfg_attr(rustfmt, rustfmt::skip)]
454macro_rules! exp {
455    // `ExpTokenPair` helper rules.
456    (@tok, $tok:ident) => {
457        $crate::parser::token_type::ExpTokenPair {
458            tok: rustc_ast::token::$tok,
459            token_type: $crate::parser::token_type::TokenType::$tok
460        }
461    };
462
463    // `ExpKeywordPair` helper rules.
464    (@kw, $kw:ident, $token_type:ident) => {
465        $crate::parser::token_type::ExpKeywordPair {
466            kw: rustc_span::symbol::kw::$kw,
467            token_type: $crate::parser::token_type::TokenType::$token_type,
468        }
469    };
470    (@sym, $kw:ident, $token_type:ident) => {
471        $crate::parser::token_type::ExpKeywordPair {
472            kw: rustc_span::symbol::sym::$kw,
473            token_type: $crate::parser::token_type::TokenType::$token_type,
474        }
475    };
476
477    (Eq)             => { exp!(@tok, Eq) };
478    (Lt)             => { exp!(@tok, Lt) };
479    (Le)             => { exp!(@tok, Le) };
480    (EqEq)           => { exp!(@tok, EqEq) };
481    (Gt)             => { exp!(@tok, Gt) };
482    (AndAnd)         => { exp!(@tok, AndAnd) };
483    (OrOr)           => { exp!(@tok, OrOr) };
484    (Bang)           => { exp!(@tok, Bang) };
485    (Tilde)          => { exp!(@tok, Tilde) };
486    (Plus)           => { exp!(@tok, Plus) };
487    (Minus)          => { exp!(@tok, Minus) };
488    (Star)           => { exp!(@tok, Star) };
489    (And)            => { exp!(@tok, And) };
490    (Or)             => { exp!(@tok, Or) };
491    (At)             => { exp!(@tok, At) };
492    (Dot)            => { exp!(@tok, Dot) };
493    (DotDot)         => { exp!(@tok, DotDot) };
494    (DotDotDot)      => { exp!(@tok, DotDotDot) };
495    (DotDotEq)       => { exp!(@tok, DotDotEq) };
496    (Comma)          => { exp!(@tok, Comma) };
497    (Semi)           => { exp!(@tok, Semi) };
498    (Colon)          => { exp!(@tok, Colon) };
499    (PathSep)        => { exp!(@tok, PathSep) };
500    (RArrow)         => { exp!(@tok, RArrow) };
501    (FatArrow)       => { exp!(@tok, FatArrow) };
502    (Pound)          => { exp!(@tok, Pound) };
503    (Question)       => { exp!(@tok, Question) };
504    (Eof)            => { exp!(@tok, Eof) };
505
506    (OpenParen)      => { exp!(@tok, OpenParen) };
507    (OpenBrace)      => { exp!(@tok, OpenBrace) };
508    (OpenBracket)    => { exp!(@tok, OpenBracket) };
509    (CloseParen)     => { exp!(@tok, CloseParen) };
510    (CloseBrace)     => { exp!(@tok, CloseBrace) };
511    (CloseBracket)   => { exp!(@tok, CloseBracket) };
512
513    (As)             => { exp!(@kw, As,         KwAs) };
514    (Async)          => { exp!(@kw, Async,      KwAsync) };
515    (Auto)           => { exp!(@kw, Auto,       KwAuto) };
516    (Await)          => { exp!(@kw, Await,      KwAwait) };
517    (Become)         => { exp!(@kw, Become,     KwBecome) };
518    (Box)            => { exp!(@kw, Box,        KwBox) };
519    (Break)          => { exp!(@kw, Break,      KwBreak) };
520    (Catch)          => { exp!(@kw, Catch,      KwCatch) };
521    (Const)          => { exp!(@kw, Const,      KwConst) };
522    (Continue)       => { exp!(@kw, Continue,   KwContinue) };
523    (ContractEnsures)  => { exp!(@kw, ContractEnsures, KwContractEnsures) };
524    (ContractRequires) => { exp!(@kw, ContractRequires, KwContractRequires) };
525    (Crate)          => { exp!(@kw, Crate,      KwCrate) };
526    (Default)        => { exp!(@kw, Default,    KwDefault) };
527    (Dyn)            => { exp!(@kw, Dyn,        KwDyn) };
528    (Else)           => { exp!(@kw, Else,       KwElse) };
529    (Enum)           => { exp!(@kw, Enum,       KwEnum) };
530    (Extern)         => { exp!(@kw, Extern,     KwExtern) };
531    (Fn)             => { exp!(@kw, Fn,         KwFn) };
532    (For)            => { exp!(@kw, For,        KwFor) };
533    (Final)          => { exp!(@kw, Final,      KwFinal) };
534    (Gen)            => { exp!(@kw, Gen,        KwGen) };
535    (If)             => { exp!(@kw, If,         KwIf) };
536    (Impl)           => { exp!(@kw, Impl,       KwImpl) };
537    (In)             => { exp!(@kw, In,         KwIn) };
538    (Let)            => { exp!(@kw, Let,        KwLet) };
539    (Loop)           => { exp!(@kw, Loop,       KwLoop) };
540    (Macro)          => { exp!(@kw, Macro,      KwMacro) };
541    (MacroRules)     => { exp!(@kw, MacroRules, KwMacroRules) };
542    (Match)          => { exp!(@kw, Match,      KwMatch) };
543    (Mod)            => { exp!(@kw, Mod,        KwMod) };
544    (Move)           => { exp!(@kw, Move,       KwMove) };
545    (Mut)            => { exp!(@kw, Mut,        KwMut) };
546    (Pin)            => { exp!(@kw, Pin,        KwMut) };
547    (Pub)            => { exp!(@kw, Pub,        KwPub) };
548    (Raw)            => { exp!(@kw, Raw,        KwRaw) };
549    (Ref)            => { exp!(@kw, Ref,        KwRef) };
550    (Return)         => { exp!(@kw, Return,     KwReturn) };
551    (Reuse)          => { exp!(@kw, Reuse,      KwReuse) };
552    (Safe)           => { exp!(@kw, Safe,       KwSafe) };
553    (SelfUpper)      => { exp!(@kw, SelfUpper,  KwSelfUpper) };
554    (Static)         => { exp!(@kw, Static,     KwStatic) };
555    (Struct)         => { exp!(@kw, Struct,     KwStruct) };
556    (Super)          => { exp!(@kw, Super,      KwSuper) };
557    (Trait)          => { exp!(@kw, Trait,      KwTrait) };
558    (Try)            => { exp!(@kw, Try,        KwTry) };
559    (Type)           => { exp!(@kw, Type,       KwType) };
560    (Underscore)     => { exp!(@kw, Underscore, KwUnderscore) };
561    (Unsafe)         => { exp!(@kw, Unsafe,     KwUnsafe) };
562    (Use)            => { exp!(@kw, Use,        KwUse) };
563    (Where)          => { exp!(@kw, Where,      KwWhere) };
564    (While)          => { exp!(@kw, While,      KwWhile) };
565    (Yield)          => { exp!(@kw, Yield,      KwYield) };
566
567    (AttSyntax)      => { exp!(@sym, att_syntax,      SymAttSyntax) };
568    (Bikeshed)       => { exp!(@sym, bikeshed,        SymBikeshed) };
569    (ClobberAbi)     => { exp!(@sym, clobber_abi,     SymClobberAbi) };
570    (Inlateout)      => { exp!(@sym, inlateout,       SymInlateout) };
571    (Inout)          => { exp!(@sym, inout,           SymInout) };
572    (Is)             => { exp!(@sym, is,              SymIs) };
573    (Label)          => { exp!(@sym, label,           SymLabel) };
574    (Lateout)        => { exp!(@sym, lateout,         SymLateout) };
575    (MayUnwind)      => { exp!(@sym, may_unwind,      SymMayUnwind) };
576    (Nomem)          => { exp!(@sym, nomem,           SymNomem) };
577    (Noreturn)       => { exp!(@sym, noreturn,        SymNoreturn) };
578    (Nostack)        => { exp!(@sym, nostack,         SymNostack) };
579    (Null)           => { exp!(@sym, null,            SymNull) };
580    (Options)        => { exp!(@sym, options,         SymOptions) };
581    (Out)            => { exp!(@sym, out,             SymOut) };
582    (Pin)            => { exp!(@sym, pin,             SymPin) };
583    (PreservesFlags) => { exp!(@sym, preserves_flags, SymPreservesFlags) };
584    (Pure)           => { exp!(@sym, pure,            SymPure) };
585    (Readonly)       => { exp!(@sym, readonly,        SymReadonly) };
586    (Sym)            => { exp!(@sym, sym,             SymSym) };
587}
588
589/// A bitset type designed specifically for `Parser::expected_token_types`,
590/// which is very hot. `u128` is the smallest integer that will fit every
591/// `TokenType` value.
592#[derive(#[automatically_derived]
#[doc(hidden)]
unsafe impl ::core::clone::TrivialClone for TokenTypeSet { }
#[automatically_derived]
impl ::core::clone::Clone for TokenTypeSet {
    #[inline]
    fn clone(&self) -> TokenTypeSet {
        let _: ::core::clone::AssertParamIsClone<u128>;
        *self
    }
}Clone, #[automatically_derived]
impl ::core::marker::Copy for TokenTypeSet { }Copy)]
593pub(super) struct TokenTypeSet(u128);
594
595impl TokenTypeSet {
596    pub(super) const fn new() -> TokenTypeSet {
597        TokenTypeSet(0)
598    }
599
600    pub(super) fn is_empty(&self) -> bool {
601        self.0 == 0
602    }
603
604    pub(super) fn insert(&mut self, token_type: TokenType) {
605        self.0 = self.0 | (1u128 << token_type as u32)
606    }
607
608    pub(super) fn clear(&mut self) {
609        self.0 = 0
610    }
611
612    pub(super) fn contains(&self, token_type: TokenType) -> bool {
613        self.0 & (1u128 << token_type as u32) != 0
614    }
615
616    pub(super) fn iter(&self) -> TokenTypeSetIter {
617        TokenTypeSetIter(*self)
618    }
619}
620
621// The `TokenTypeSet` is a copy of the set being iterated. It initially holds
622// the entire set. Each bit is cleared as it is returned. We have finished once
623// it is all zeroes.
624pub(super) struct TokenTypeSetIter(TokenTypeSet);
625
626impl Iterator for TokenTypeSetIter {
627    type Item = TokenType;
628
629    fn next(&mut self) -> Option<TokenType> {
630        let num_bits: u32 = (size_of_val(&self.0.0) * 8) as u32;
631        {
    match (&num_bits, &128) {
        (left_val, right_val) => {
            if !(*left_val == *right_val) {
                let kind = ::core::panicking::AssertKind::Eq;
                ::core::panicking::assert_failed(kind, &*left_val,
                    &*right_val, ::core::option::Option::None);
            }
        }
    }
};assert_eq!(num_bits, 128);
632        let z = self.0.0.trailing_zeros();
633        if z == num_bits {
634            None
635        } else {
636            self.0.0 &= !(1 << z); // clear the trailing 1 bit
637            Some(TokenType::from_u32(z))
638        }
639    }
640}
rustc_parse/parser/token_type.rs

rustc_parse/parser/
token_type.rs
