rustc_parse/parser/
token_type.rs

1use rustc_ast::token::TokenKind;
2use rustc_span::symbol::{Symbol, kw, sym};
3
4/// Used in "expected"/"expected one of" error messages. Tokens are added here
5/// as necessary. Tokens with values (e.g. literals, identifiers) are
6/// represented by a single variant (e.g. `Literal`, `Ident`).
7///
8/// It's an awkward representation, but it's important for performance. It's a
9/// C-style parameterless enum so that `TokenTypeSet` can be a bitset. This is
10/// important because `Parser::expected_token_types` is very hot. `TokenType`
11/// used to have variants with parameters (e.g. all the keywords were in a
12/// single `Keyword` variant with a `Symbol` parameter) and
13/// `Parser::expected_token_types` was a `Vec<TokenType>` which was much slower
14/// to manipulate.
15///
16/// We really want to keep the number of variants to 128 or fewer, so that
17/// `TokenTypeSet` can be implemented with a `u128`.
18#[derive(#[automatically_derived]
impl ::core::fmt::Debug for TokenType {
    #[inline]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::write_str(f,
            match self {
                TokenType::Eq => "Eq",
                TokenType::Lt => "Lt",
                TokenType::Le => "Le",
                TokenType::EqEq => "EqEq",
                TokenType::Gt => "Gt",
                TokenType::AndAnd => "AndAnd",
                TokenType::OrOr => "OrOr",
                TokenType::Bang => "Bang",
                TokenType::Tilde => "Tilde",
                TokenType::Plus => "Plus",
                TokenType::Minus => "Minus",
                TokenType::Star => "Star",
                TokenType::And => "And",
                TokenType::Or => "Or",
                TokenType::At => "At",
                TokenType::Dot => "Dot",
                TokenType::DotDot => "DotDot",
                TokenType::DotDotDot => "DotDotDot",
                TokenType::DotDotEq => "DotDotEq",
                TokenType::Comma => "Comma",
                TokenType::Semi => "Semi",
                TokenType::Colon => "Colon",
                TokenType::PathSep => "PathSep",
                TokenType::RArrow => "RArrow",
                TokenType::FatArrow => "FatArrow",
                TokenType::Pound => "Pound",
                TokenType::Question => "Question",
                TokenType::OpenParen => "OpenParen",
                TokenType::CloseParen => "CloseParen",
                TokenType::OpenBrace => "OpenBrace",
                TokenType::CloseBrace => "CloseBrace",
                TokenType::OpenBracket => "OpenBracket",
                TokenType::CloseBracket => "CloseBracket",
                TokenType::Eof => "Eof",
                TokenType::Operator => "Operator",
                TokenType::Ident => "Ident",
                TokenType::Lifetime => "Lifetime",
                TokenType::Path => "Path",
                TokenType::Type => "Type",
                TokenType::Const => "Const",
                TokenType::KwAs => "KwAs",
                TokenType::KwAsync => "KwAsync",
                TokenType::KwAuto => "KwAuto",
                TokenType::KwAwait => "KwAwait",
                TokenType::KwBecome => "KwBecome",
                TokenType::KwBox => "KwBox",
                TokenType::KwBreak => "KwBreak",
                TokenType::KwCatch => "KwCatch",
                TokenType::KwConst => "KwConst",
                TokenType::KwContinue => "KwContinue",
                TokenType::KwContractEnsures => "KwContractEnsures",
                TokenType::KwContractRequires => "KwContractRequires",
                TokenType::KwCrate => "KwCrate",
                TokenType::KwDefault => "KwDefault",
                TokenType::KwDyn => "KwDyn",
                TokenType::KwElse => "KwElse",
                TokenType::KwEnum => "KwEnum",
                TokenType::KwExtern => "KwExtern",
                TokenType::KwFn => "KwFn",
                TokenType::KwFor => "KwFor",
                TokenType::KwGen => "KwGen",
                TokenType::KwIf => "KwIf",
                TokenType::KwImpl => "KwImpl",
                TokenType::KwIn => "KwIn",
                TokenType::KwLet => "KwLet",
                TokenType::KwLoop => "KwLoop",
                TokenType::KwMacro => "KwMacro",
                TokenType::KwMacroRules => "KwMacroRules",
                TokenType::KwMatch => "KwMatch",
                TokenType::KwMod => "KwMod",
                TokenType::KwMove => "KwMove",
                TokenType::KwMut => "KwMut",
                TokenType::KwPub => "KwPub",
                TokenType::KwRaw => "KwRaw",
                TokenType::KwRef => "KwRef",
                TokenType::KwReturn => "KwReturn",
                TokenType::KwReuse => "KwReuse",
                TokenType::KwSafe => "KwSafe",
                TokenType::KwSelfUpper => "KwSelfUpper",
                TokenType::KwStatic => "KwStatic",
                TokenType::KwStruct => "KwStruct",
                TokenType::KwSuper => "KwSuper",
                TokenType::KwTrait => "KwTrait",
                TokenType::KwTry => "KwTry",
                TokenType::KwType => "KwType",
                TokenType::KwUnderscore => "KwUnderscore",
                TokenType::KwUnsafe => "KwUnsafe",
                TokenType::KwUse => "KwUse",
                TokenType::KwWhere => "KwWhere",
                TokenType::KwWhile => "KwWhile",
                TokenType::KwYield => "KwYield",
                TokenType::SymAttSyntax => "SymAttSyntax",
                TokenType::SymBikeshed => "SymBikeshed",
                TokenType::SymClobberAbi => "SymClobberAbi",
                TokenType::SymInlateout => "SymInlateout",
                TokenType::SymInout => "SymInout",
                TokenType::SymIs => "SymIs",
                TokenType::SymLabel => "SymLabel",
                TokenType::SymLateout => "SymLateout",
                TokenType::SymMayUnwind => "SymMayUnwind",
                TokenType::SymNomem => "SymNomem",
                TokenType::SymNoreturn => "SymNoreturn",
                TokenType::SymNostack => "SymNostack",
                TokenType::SymNull => "SymNull",
                TokenType::SymOptions => "SymOptions",
                TokenType::SymOut => "SymOut",
                TokenType::SymPin => "SymPin",
                TokenType::SymPreservesFlags => "SymPreservesFlags",
                TokenType::SymPure => "SymPure",
                TokenType::SymReadonly => "SymReadonly",
                TokenType::SymSym => "SymSym",
            })
    }
}Debug, #[automatically_derived]
impl ::core::clone::Clone for TokenType {
    #[inline]
    fn clone(&self) -> TokenType { *self }
}Clone, #[automatically_derived]
impl ::core::marker::Copy for TokenType { }Copy, #[automatically_derived]
impl ::core::cmp::PartialEq for TokenType {
    #[inline]
    fn eq(&self, other: &TokenType) -> bool {
        let __self_discr = ::core::intrinsics::discriminant_value(self);
        let __arg1_discr = ::core::intrinsics::discriminant_value(other);
        __self_discr == __arg1_discr
    }
}PartialEq)]
19pub enum TokenType {
20    // Expression-operator symbols
21    Eq,
22    Lt,
23    Le,
24    EqEq,
25    Gt,
26    AndAnd,
27    OrOr,
28    Bang,
29    Tilde,
30
31    // BinOps
32    Plus,
33    Minus,
34    Star,
35    And,
36    Or,
37
38    // Structural symbols
39    At,
40    Dot,
41    DotDot,
42    DotDotDot,
43    DotDotEq,
44    Comma,
45    Semi,
46    Colon,
47    PathSep,
48    RArrow,
49    FatArrow,
50    Pound,
51    Question,
52    OpenParen,
53    CloseParen,
54    OpenBrace,
55    CloseBrace,
56    OpenBracket,
57    CloseBracket,
58    Eof,
59
60    // Token types with some details elided.
61    /// Any operator.
62    Operator,
63    /// Any identifier token.
64    Ident,
65    /// Any lifetime token.
66    Lifetime,
67    /// Any token that can start a path.
68    Path,
69    /// Any token that can start a type.
70    Type,
71    /// Any token that can start a const expression.
72    Const,
73
74    // Keywords
75    // tidy-alphabetical-start
76    KwAs,
77    KwAsync,
78    KwAuto,
79    KwAwait,
80    KwBecome,
81    KwBox,
82    KwBreak,
83    KwCatch,
84    KwConst,
85    KwContinue,
86    KwContractEnsures,
87    KwContractRequires,
88    KwCrate,
89    KwDefault,
90    KwDyn,
91    KwElse,
92    KwEnum,
93    KwExtern,
94    KwFn,
95    KwFor,
96    KwGen,
97    KwIf,
98    KwImpl,
99    KwIn,
100    KwLet,
101    KwLoop,
102    KwMacro,
103    KwMacroRules,
104    KwMatch,
105    KwMod,
106    KwMove,
107    KwMut,
108    KwPub,
109    KwRaw,
110    KwRef,
111    KwReturn,
112    KwReuse,
113    KwSafe,
114    KwSelfUpper,
115    KwStatic,
116    KwStruct,
117    KwSuper,
118    KwTrait,
119    KwTry,
120    KwType,
121    KwUnderscore,
122    KwUnsafe,
123    KwUse,
124    KwWhere,
125    KwWhile,
126    KwYield,
127    // tidy-alphabetical-end
128
129    // Keyword-like symbols.
130    // tidy-alphabetical-start
131    SymAttSyntax,
132    SymBikeshed,
133    SymClobberAbi,
134    SymInlateout,
135    SymInout,
136    SymIs,
137    SymLabel,
138    SymLateout,
139    SymMayUnwind,
140    SymNomem,
141    SymNoreturn,
142    SymNostack,
143    SymNull,
144    SymOptions,
145    SymOut,
146    SymPin,
147    SymPreservesFlags,
148    SymPure,
149    SymReadonly,
150    SymSym,
151    // tidy-alphabetical-end
152}
153
154// Macro to avoid repetitive boilerplate code.
155macro_rules! from_u32_match {
156    ($val:ident; $($tok:ident,)+) => {
157        // A more obvious formulation would be `0 => TokenType::Eq`. But
158        // this formulation with the guard lets us avoid specifying a
159        // specific integer for each variant.
160        match $val {
161            $(
162                t if t == TokenType::$tok as u32 => TokenType::$tok,
163            )+
164            _ => panic!("unhandled value: {}", $val),
165        }
166    };
167}
168
169impl TokenType {
170    fn from_u32(val: u32) -> TokenType {
171        let token_type = match val {
    t if t == TokenType::Eq as u32 => TokenType::Eq,
    t if t == TokenType::Lt as u32 => TokenType::Lt,
    t if t == TokenType::Le as u32 => TokenType::Le,
    t if t == TokenType::EqEq as u32 => TokenType::EqEq,
    t if t == TokenType::Gt as u32 => TokenType::Gt,
    t if t == TokenType::AndAnd as u32 => TokenType::AndAnd,
    t if t == TokenType::OrOr as u32 => TokenType::OrOr,
    t if t == TokenType::Bang as u32 => TokenType::Bang,
    t if t == TokenType::Tilde as u32 => TokenType::Tilde,
    t if t == TokenType::Plus as u32 => TokenType::Plus,
    t if t == TokenType::Minus as u32 => TokenType::Minus,
    t if t == TokenType::Star as u32 => TokenType::Star,
    t if t == TokenType::And as u32 => TokenType::And,
    t if t == TokenType::Or as u32 => TokenType::Or,
    t if t == TokenType::At as u32 => TokenType::At,
    t if t == TokenType::Dot as u32 => TokenType::Dot,
    t if t == TokenType::DotDot as u32 => TokenType::DotDot,
    t if t == TokenType::DotDotDot as u32 => TokenType::DotDotDot,
    t if t == TokenType::DotDotEq as u32 => TokenType::DotDotEq,
    t if t == TokenType::Comma as u32 => TokenType::Comma,
    t if t == TokenType::Semi as u32 => TokenType::Semi,
    t if t == TokenType::Colon as u32 => TokenType::Colon,
    t if t == TokenType::PathSep as u32 => TokenType::PathSep,
    t if t == TokenType::RArrow as u32 => TokenType::RArrow,
    t if t == TokenType::FatArrow as u32 => TokenType::FatArrow,
    t if t == TokenType::Pound as u32 => TokenType::Pound,
    t if t == TokenType::Question as u32 => TokenType::Question,
    t if t == TokenType::OpenParen as u32 => TokenType::OpenParen,
    t if t == TokenType::CloseParen as u32 => TokenType::CloseParen,
    t if t == TokenType::OpenBrace as u32 => TokenType::OpenBrace,
    t if t == TokenType::CloseBrace as u32 => TokenType::CloseBrace,
    t if t == TokenType::OpenBracket as u32 => TokenType::OpenBracket,
    t if t == TokenType::CloseBracket as u32 => TokenType::CloseBracket,
    t if t == TokenType::Eof as u32 => TokenType::Eof,
    t if t == TokenType::Operator as u32 => TokenType::Operator,
    t if t == TokenType::Ident as u32 => TokenType::Ident,
    t if t == TokenType::Lifetime as u32 => TokenType::Lifetime,
    t if t == TokenType::Path as u32 => TokenType::Path,
    t if t == TokenType::Type as u32 => TokenType::Type,
    t if t == TokenType::Const as u32 => TokenType::Const,
    t if t == TokenType::KwAs as u32 => TokenType::KwAs,
    t if t == TokenType::KwAsync as u32 => TokenType::KwAsync,
    t if t == TokenType::KwAuto as u32 => TokenType::KwAuto,
    t if t == TokenType::KwAwait as u32 => TokenType::KwAwait,
    t if t == TokenType::KwBecome as u32 => TokenType::KwBecome,
    t if t == TokenType::KwBox as u32 => TokenType::KwBox,
    t if t == TokenType::KwBreak as u32 => TokenType::KwBreak,
    t if t == TokenType::KwCatch as u32 => TokenType::KwCatch,
    t if t == TokenType::KwConst as u32 => TokenType::KwConst,
    t if t == TokenType::KwContinue as u32 => TokenType::KwContinue,
    t if t == TokenType::KwContractEnsures as u32 =>
        TokenType::KwContractEnsures,
    t if t == TokenType::KwContractRequires as u32 =>
        TokenType::KwContractRequires,
    t if t == TokenType::KwCrate as u32 => TokenType::KwCrate,
    t if t == TokenType::KwDefault as u32 => TokenType::KwDefault,
    t if t == TokenType::KwDyn as u32 => TokenType::KwDyn,
    t if t == TokenType::KwElse as u32 => TokenType::KwElse,
    t if t == TokenType::KwEnum as u32 => TokenType::KwEnum,
    t if t == TokenType::KwExtern as u32 => TokenType::KwExtern,
    t if t == TokenType::KwFn as u32 => TokenType::KwFn,
    t if t == TokenType::KwFor as u32 => TokenType::KwFor,
    t if t == TokenType::KwGen as u32 => TokenType::KwGen,
    t if t == TokenType::KwIf as u32 => TokenType::KwIf,
    t if t == TokenType::KwImpl as u32 => TokenType::KwImpl,
    t if t == TokenType::KwIn as u32 => TokenType::KwIn,
    t if t == TokenType::KwLet as u32 => TokenType::KwLet,
    t if t == TokenType::KwLoop as u32 => TokenType::KwLoop,
    t if t == TokenType::KwMacro as u32 => TokenType::KwMacro,
    t if t == TokenType::KwMacroRules as u32 => TokenType::KwMacroRules,
    t if t == TokenType::KwMatch as u32 => TokenType::KwMatch,
    t if t == TokenType::KwMod as u32 => TokenType::KwMod,
    t if t == TokenType::KwMove as u32 => TokenType::KwMove,
    t if t == TokenType::KwMut as u32 => TokenType::KwMut,
    t if t == TokenType::KwPub as u32 => TokenType::KwPub,
    t if t == TokenType::KwRaw as u32 => TokenType::KwRaw,
    t if t == TokenType::KwRef as u32 => TokenType::KwRef,
    t if t == TokenType::KwReturn as u32 => TokenType::KwReturn,
    t if t == TokenType::KwReuse as u32 => TokenType::KwReuse,
    t if t == TokenType::KwSafe as u32 => TokenType::KwSafe,
    t if t == TokenType::KwSelfUpper as u32 => TokenType::KwSelfUpper,
    t if t == TokenType::KwStatic as u32 => TokenType::KwStatic,
    t if t == TokenType::KwStruct as u32 => TokenType::KwStruct,
    t if t == TokenType::KwSuper as u32 => TokenType::KwSuper,
    t if t == TokenType::KwTrait as u32 => TokenType::KwTrait,
    t if t == TokenType::KwTry as u32 => TokenType::KwTry,
    t if t == TokenType::KwType as u32 => TokenType::KwType,
    t if t == TokenType::KwUnderscore as u32 => TokenType::KwUnderscore,
    t if t == TokenType::KwUnsafe as u32 => TokenType::KwUnsafe,
    t if t == TokenType::KwUse as u32 => TokenType::KwUse,
    t if t == TokenType::KwWhere as u32 => TokenType::KwWhere,
    t if t == TokenType::KwWhile as u32 => TokenType::KwWhile,
    t if t == TokenType::KwYield as u32 => TokenType::KwYield,
    t if t == TokenType::SymAttSyntax as u32 => TokenType::SymAttSyntax,
    t if t == TokenType::SymClobberAbi as u32 => TokenType::SymClobberAbi,
    t if t == TokenType::SymInlateout as u32 => TokenType::SymInlateout,
    t if t == TokenType::SymInout as u32 => TokenType::SymInout,
    t if t == TokenType::SymIs as u32 => TokenType::SymIs,
    t if t == TokenType::SymLabel as u32 => TokenType::SymLabel,
    t if t == TokenType::SymLateout as u32 => TokenType::SymLateout,
    t if t == TokenType::SymMayUnwind as u32 => TokenType::SymMayUnwind,
    t if t == TokenType::SymNomem as u32 => TokenType::SymNomem,
    t if t == TokenType::SymNoreturn as u32 => TokenType::SymNoreturn,
    t if t == TokenType::SymNostack as u32 => TokenType::SymNostack,
    t if t == TokenType::SymNull as u32 => TokenType::SymNull,
    t if t == TokenType::SymOptions as u32 => TokenType::SymOptions,
    t if t == TokenType::SymOut as u32 => TokenType::SymOut,
    t if t == TokenType::SymPreservesFlags as u32 =>
        TokenType::SymPreservesFlags,
    t if t == TokenType::SymPure as u32 => TokenType::SymPure,
    t if t == TokenType::SymReadonly as u32 => TokenType::SymReadonly,
    t if t == TokenType::SymSym as u32 => TokenType::SymSym,
    _ => {
        ::core::panicking::panic_fmt(format_args!("unhandled value: {0}",
                val));
    }
}from_u32_match! { val;
172            Eq,
173            Lt,
174            Le,
175            EqEq,
176            Gt,
177            AndAnd,
178            OrOr,
179            Bang,
180            Tilde,
181
182            Plus,
183            Minus,
184            Star,
185            And,
186            Or,
187
188            At,
189            Dot,
190            DotDot,
191            DotDotDot,
192            DotDotEq,
193            Comma,
194            Semi,
195            Colon,
196            PathSep,
197            RArrow,
198            FatArrow,
199            Pound,
200            Question,
201            OpenParen,
202            CloseParen,
203            OpenBrace,
204            CloseBrace,
205            OpenBracket,
206            CloseBracket,
207            Eof,
208
209            Operator,
210            Ident,
211            Lifetime,
212            Path,
213            Type,
214            Const,
215
216            KwAs,
217            KwAsync,
218            KwAuto,
219            KwAwait,
220            KwBecome,
221            KwBox,
222            KwBreak,
223            KwCatch,
224            KwConst,
225            KwContinue,
226            KwContractEnsures,
227            KwContractRequires,
228            KwCrate,
229            KwDefault,
230            KwDyn,
231            KwElse,
232            KwEnum,
233            KwExtern,
234            KwFn,
235            KwFor,
236            KwGen,
237            KwIf,
238            KwImpl,
239            KwIn,
240            KwLet,
241            KwLoop,
242            KwMacro,
243            KwMacroRules,
244            KwMatch,
245            KwMod,
246            KwMove,
247            KwMut,
248            KwPub,
249            KwRaw,
250            KwRef,
251            KwReturn,
252            KwReuse,
253            KwSafe,
254            KwSelfUpper,
255            KwStatic,
256            KwStruct,
257            KwSuper,
258            KwTrait,
259            KwTry,
260            KwType,
261            KwUnderscore,
262            KwUnsafe,
263            KwUse,
264            KwWhere,
265            KwWhile,
266            KwYield,
267
268            SymAttSyntax,
269            SymClobberAbi,
270            SymInlateout,
271            SymInout,
272            SymIs,
273            SymLabel,
274            SymLateout,
275            SymMayUnwind,
276            SymNomem,
277            SymNoreturn,
278            SymNostack,
279            SymNull,
280            SymOptions,
281            SymOut,
282            SymPreservesFlags,
283            SymPure,
284            SymReadonly,
285            SymSym,
286        };
287        token_type
288    }
289
290    pub(super) fn is_keyword(&self) -> Option<Symbol> {
291        match self {
292            TokenType::KwAs => Some(kw::As),
293            TokenType::KwAsync => Some(kw::Async),
294            TokenType::KwAuto => Some(kw::Auto),
295            TokenType::KwAwait => Some(kw::Await),
296            TokenType::KwBecome => Some(kw::Become),
297            TokenType::KwBox => Some(kw::Box),
298            TokenType::KwBreak => Some(kw::Break),
299            TokenType::KwCatch => Some(kw::Catch),
300            TokenType::KwConst => Some(kw::Const),
301            TokenType::KwContinue => Some(kw::Continue),
302            TokenType::KwContractEnsures => Some(kw::ContractEnsures),
303            TokenType::KwContractRequires => Some(kw::ContractRequires),
304            TokenType::KwCrate => Some(kw::Crate),
305            TokenType::KwDefault => Some(kw::Default),
306            TokenType::KwDyn => Some(kw::Dyn),
307            TokenType::KwElse => Some(kw::Else),
308            TokenType::KwEnum => Some(kw::Enum),
309            TokenType::KwExtern => Some(kw::Extern),
310            TokenType::KwFn => Some(kw::Fn),
311            TokenType::KwFor => Some(kw::For),
312            TokenType::KwGen => Some(kw::Gen),
313            TokenType::KwIf => Some(kw::If),
314            TokenType::KwImpl => Some(kw::Impl),
315            TokenType::KwIn => Some(kw::In),
316            TokenType::KwLet => Some(kw::Let),
317            TokenType::KwLoop => Some(kw::Loop),
318            TokenType::KwMacroRules => Some(kw::MacroRules),
319            TokenType::KwMacro => Some(kw::Macro),
320            TokenType::KwMatch => Some(kw::Match),
321            TokenType::KwMod => Some(kw::Mod),
322            TokenType::KwMove => Some(kw::Move),
323            TokenType::KwMut => Some(kw::Mut),
324            TokenType::KwPub => Some(kw::Pub),
325            TokenType::KwRaw => Some(kw::Raw),
326            TokenType::KwRef => Some(kw::Ref),
327            TokenType::KwReturn => Some(kw::Return),
328            TokenType::KwReuse => Some(kw::Reuse),
329            TokenType::KwSafe => Some(kw::Safe),
330            TokenType::KwSelfUpper => Some(kw::SelfUpper),
331            TokenType::KwStatic => Some(kw::Static),
332            TokenType::KwStruct => Some(kw::Struct),
333            TokenType::KwSuper => Some(kw::Super),
334            TokenType::KwTrait => Some(kw::Trait),
335            TokenType::KwTry => Some(kw::Try),
336            TokenType::KwType => Some(kw::Type),
337            TokenType::KwUnderscore => Some(kw::Underscore),
338            TokenType::KwUnsafe => Some(kw::Unsafe),
339            TokenType::KwUse => Some(kw::Use),
340            TokenType::KwWhere => Some(kw::Where),
341            TokenType::KwWhile => Some(kw::While),
342            TokenType::KwYield => Some(kw::Yield),
343
344            TokenType::SymAttSyntax => Some(sym::att_syntax),
345            TokenType::SymClobberAbi => Some(sym::clobber_abi),
346            TokenType::SymInlateout => Some(sym::inlateout),
347            TokenType::SymInout => Some(sym::inout),
348            TokenType::SymIs => Some(sym::is),
349            TokenType::SymLabel => Some(sym::label),
350            TokenType::SymLateout => Some(sym::lateout),
351            TokenType::SymMayUnwind => Some(sym::may_unwind),
352            TokenType::SymNomem => Some(sym::nomem),
353            TokenType::SymNoreturn => Some(sym::noreturn),
354            TokenType::SymNostack => Some(sym::nostack),
355            TokenType::SymNull => Some(sym::null),
356            TokenType::SymOptions => Some(sym::options),
357            TokenType::SymOut => Some(sym::out),
358            TokenType::SymPreservesFlags => Some(sym::preserves_flags),
359            TokenType::SymPure => Some(sym::pure),
360            TokenType::SymReadonly => Some(sym::readonly),
361            TokenType::SymSym => Some(sym::sym),
362            _ => None,
363        }
364    }
365
366    // The output should be the same as that produced by
367    // `rustc_ast_pretty::pprust::token_to_string`.
368    pub(super) fn to_string(&self) -> String {
369        match self {
370            TokenType::Eq => "`=`",
371            TokenType::Lt => "`<`",
372            TokenType::Le => "`<=`",
373            TokenType::EqEq => "`==`",
374            TokenType::Gt => "`>`",
375            TokenType::AndAnd => "`&&`",
376            TokenType::OrOr => "`||`",
377            TokenType::Bang => "`!`",
378            TokenType::Tilde => "`~`",
379
380            TokenType::Plus => "`+`",
381            TokenType::Minus => "`-`",
382            TokenType::Star => "`*`",
383            TokenType::And => "`&`",
384            TokenType::Or => "`|`",
385
386            TokenType::At => "`@`",
387            TokenType::Dot => "`.`",
388            TokenType::DotDot => "`..`",
389            TokenType::DotDotDot => "`...`",
390            TokenType::DotDotEq => "`..=`",
391            TokenType::Comma => "`,`",
392            TokenType::Semi => "`;`",
393            TokenType::Colon => "`:`",
394            TokenType::PathSep => "`::`",
395            TokenType::RArrow => "`->`",
396            TokenType::FatArrow => "`=>`",
397            TokenType::Pound => "`#`",
398            TokenType::Question => "`?`",
399            TokenType::OpenParen => "`(`",
400            TokenType::CloseParen => "`)`",
401            TokenType::OpenBrace => "`{`",
402            TokenType::CloseBrace => "`}`",
403            TokenType::OpenBracket => "`[`",
404            TokenType::CloseBracket => "`]`",
405            TokenType::Eof => "<eof>",
406
407            TokenType::Operator => "an operator",
408            TokenType::Ident => "identifier",
409            TokenType::Lifetime => "lifetime",
410            TokenType::Path => "path",
411            TokenType::Type => "type",
412            TokenType::Const => "a const expression",
413
414            _ => return ::alloc::__export::must_use({
        ::alloc::fmt::format(format_args!("`{0}`",
                self.is_keyword().unwrap()))
    })format!("`{}`", self.is_keyword().unwrap()),
415        }
416        .to_string()
417    }
418}
419
420/// Used by various `Parser` methods such as `check` and `eat`. The first field
421/// is always by used those methods. The second field is only used when the
422/// first field doesn't match.
423#[derive(#[automatically_derived]
impl ::core::clone::Clone for ExpTokenPair {
    #[inline]
    fn clone(&self) -> ExpTokenPair {
        let _: ::core::clone::AssertParamIsClone<TokenKind>;
        let _: ::core::clone::AssertParamIsClone<TokenType>;
        *self
    }
}Clone, #[automatically_derived]
impl ::core::marker::Copy for ExpTokenPair { }Copy, #[automatically_derived]
impl ::core::fmt::Debug for ExpTokenPair {
    #[inline]
    fn fmt(&self, f: &mut ::core::fmt::Formatter) -> ::core::fmt::Result {
        ::core::fmt::Formatter::debug_struct_field2_finish(f, "ExpTokenPair",
            "tok", &self.tok, "token_type", &&self.token_type)
    }
}Debug)]
424pub struct ExpTokenPair {
425    pub tok: TokenKind,
426    pub token_type: TokenType,
427}
428
429/// Used by various `Parser` methods such as `check_keyword` and `eat_keyword`.
430/// The first field is always used by those methods. The second field is only
431/// used when the first field doesn't match.
432#[derive(#[automatically_derived]
impl ::core::clone::Clone for ExpKeywordPair {
    #[inline]
    fn clone(&self) -> ExpKeywordPair {
        let _: ::core::clone::AssertParamIsClone<Symbol>;
        let _: ::core::clone::AssertParamIsClone<TokenType>;
        *self
    }
}Clone, #[automatically_derived]
impl ::core::marker::Copy for ExpKeywordPair { }Copy)]
433pub struct ExpKeywordPair {
434    pub kw: Symbol,
435    pub token_type: TokenType,
436}
437
438// Gets a statically-known `ExpTokenPair` pair (for non-keywords) or
439// `ExpKeywordPair` (for keywords), as used with various `check`/`expect`
440// methods in `Parser`.
441//
442// The name is short because it's used a lot.
443#[macro_export]
444// We don't use the normal `#[rustfmt::skip]` here because that triggers a
445// bogus "macro-expanded `macro_export` macros from the current crate cannot be
446// referred to by absolute paths" error, ugh. See #52234.
447#[cfg_attr(rustfmt, rustfmt::skip)]
448macro_rules! exp {
449    // `ExpTokenPair` helper rules.
450    (@tok, $tok:ident) => {
451        $crate::parser::token_type::ExpTokenPair {
452            tok: rustc_ast::token::$tok,
453            token_type: $crate::parser::token_type::TokenType::$tok
454        }
455    };
456
457    // `ExpKeywordPair` helper rules.
458    (@kw, $kw:ident, $token_type:ident) => {
459        $crate::parser::token_type::ExpKeywordPair {
460            kw: rustc_span::symbol::kw::$kw,
461            token_type: $crate::parser::token_type::TokenType::$token_type,
462        }
463    };
464    (@sym, $kw:ident, $token_type:ident) => {
465        $crate::parser::token_type::ExpKeywordPair {
466            kw: rustc_span::symbol::sym::$kw,
467            token_type: $crate::parser::token_type::TokenType::$token_type,
468        }
469    };
470
471    (Eq)             => { exp!(@tok, Eq) };
472    (Lt)             => { exp!(@tok, Lt) };
473    (Le)             => { exp!(@tok, Le) };
474    (EqEq)           => { exp!(@tok, EqEq) };
475    (Gt)             => { exp!(@tok, Gt) };
476    (AndAnd)         => { exp!(@tok, AndAnd) };
477    (OrOr)           => { exp!(@tok, OrOr) };
478    (Bang)           => { exp!(@tok, Bang) };
479    (Tilde)          => { exp!(@tok, Tilde) };
480    (Plus)           => { exp!(@tok, Plus) };
481    (Minus)          => { exp!(@tok, Minus) };
482    (Star)           => { exp!(@tok, Star) };
483    (And)            => { exp!(@tok, And) };
484    (Or)             => { exp!(@tok, Or) };
485    (At)             => { exp!(@tok, At) };
486    (Dot)            => { exp!(@tok, Dot) };
487    (DotDot)         => { exp!(@tok, DotDot) };
488    (DotDotDot)      => { exp!(@tok, DotDotDot) };
489    (DotDotEq)       => { exp!(@tok, DotDotEq) };
490    (Comma)          => { exp!(@tok, Comma) };
491    (Semi)           => { exp!(@tok, Semi) };
492    (Colon)          => { exp!(@tok, Colon) };
493    (PathSep)        => { exp!(@tok, PathSep) };
494    (RArrow)         => { exp!(@tok, RArrow) };
495    (FatArrow)       => { exp!(@tok, FatArrow) };
496    (Pound)          => { exp!(@tok, Pound) };
497    (Question)       => { exp!(@tok, Question) };
498    (Eof)            => { exp!(@tok, Eof) };
499
500    (OpenParen)      => { exp!(@tok, OpenParen) };
501    (OpenBrace)      => { exp!(@tok, OpenBrace) };
502    (OpenBracket)    => { exp!(@tok, OpenBracket) };
503    (CloseParen)     => { exp!(@tok, CloseParen) };
504    (CloseBrace)     => { exp!(@tok, CloseBrace) };
505    (CloseBracket)   => { exp!(@tok, CloseBracket) };
506
507    (As)             => { exp!(@kw, As,         KwAs) };
508    (Async)          => { exp!(@kw, Async,      KwAsync) };
509    (Auto)           => { exp!(@kw, Auto,       KwAuto) };
510    (Await)          => { exp!(@kw, Await,      KwAwait) };
511    (Become)         => { exp!(@kw, Become,     KwBecome) };
512    (Box)            => { exp!(@kw, Box,        KwBox) };
513    (Break)          => { exp!(@kw, Break,      KwBreak) };
514    (Catch)          => { exp!(@kw, Catch,      KwCatch) };
515    (Const)          => { exp!(@kw, Const,      KwConst) };
516    (Continue)       => { exp!(@kw, Continue,   KwContinue) };
517    (ContractEnsures)  => { exp!(@kw, ContractEnsures, KwContractEnsures) };
518    (ContractRequires) => { exp!(@kw, ContractRequires, KwContractRequires) };
519    (Crate)          => { exp!(@kw, Crate,      KwCrate) };
520    (Default)        => { exp!(@kw, Default,    KwDefault) };
521    (Dyn)            => { exp!(@kw, Dyn,        KwDyn) };
522    (Else)           => { exp!(@kw, Else,       KwElse) };
523    (Enum)           => { exp!(@kw, Enum,       KwEnum) };
524    (Extern)         => { exp!(@kw, Extern,     KwExtern) };
525    (Fn)             => { exp!(@kw, Fn,         KwFn) };
526    (For)            => { exp!(@kw, For,        KwFor) };
527    (Gen)            => { exp!(@kw, Gen,        KwGen) };
528    (If)             => { exp!(@kw, If,         KwIf) };
529    (Impl)           => { exp!(@kw, Impl,       KwImpl) };
530    (In)             => { exp!(@kw, In,         KwIn) };
531    (Let)            => { exp!(@kw, Let,        KwLet) };
532    (Loop)           => { exp!(@kw, Loop,       KwLoop) };
533    (Macro)          => { exp!(@kw, Macro,      KwMacro) };
534    (MacroRules)     => { exp!(@kw, MacroRules, KwMacroRules) };
535    (Match)          => { exp!(@kw, Match,      KwMatch) };
536    (Mod)            => { exp!(@kw, Mod,        KwMod) };
537    (Move)           => { exp!(@kw, Move,       KwMove) };
538    (Mut)            => { exp!(@kw, Mut,        KwMut) };
539    (Pub)            => { exp!(@kw, Pub,        KwPub) };
540    (Raw)            => { exp!(@kw, Raw,        KwRaw) };
541    (Ref)            => { exp!(@kw, Ref,        KwRef) };
542    (Return)         => { exp!(@kw, Return,     KwReturn) };
543    (Reuse)          => { exp!(@kw, Reuse,      KwReuse) };
544    (Safe)           => { exp!(@kw, Safe,       KwSafe) };
545    (SelfUpper)      => { exp!(@kw, SelfUpper,  KwSelfUpper) };
546    (Static)         => { exp!(@kw, Static,     KwStatic) };
547    (Struct)         => { exp!(@kw, Struct,     KwStruct) };
548    (Super)          => { exp!(@kw, Super,      KwSuper) };
549    (Trait)          => { exp!(@kw, Trait,      KwTrait) };
550    (Try)            => { exp!(@kw, Try,        KwTry) };
551    (Type)           => { exp!(@kw, Type,       KwType) };
552    (Underscore)     => { exp!(@kw, Underscore, KwUnderscore) };
553    (Unsafe)         => { exp!(@kw, Unsafe,     KwUnsafe) };
554    (Use)            => { exp!(@kw, Use,        KwUse) };
555    (Where)          => { exp!(@kw, Where,      KwWhere) };
556    (While)          => { exp!(@kw, While,      KwWhile) };
557    (Yield)          => { exp!(@kw, Yield,      KwYield) };
558
559    (AttSyntax)      => { exp!(@sym, att_syntax,      SymAttSyntax) };
560    (Bikeshed)       => { exp!(@sym, bikeshed,        SymBikeshed) };
561    (ClobberAbi)     => { exp!(@sym, clobber_abi,     SymClobberAbi) };
562    (Inlateout)      => { exp!(@sym, inlateout,       SymInlateout) };
563    (Inout)          => { exp!(@sym, inout,           SymInout) };
564    (Is)             => { exp!(@sym, is,              SymIs) };
565    (Label)          => { exp!(@sym, label,           SymLabel) };
566    (Lateout)        => { exp!(@sym, lateout,         SymLateout) };
567    (MayUnwind)      => { exp!(@sym, may_unwind,      SymMayUnwind) };
568    (Nomem)          => { exp!(@sym, nomem,           SymNomem) };
569    (Noreturn)       => { exp!(@sym, noreturn,        SymNoreturn) };
570    (Nostack)        => { exp!(@sym, nostack,         SymNostack) };
571    (Null)           => { exp!(@sym, null,            SymNull) };
572    (Options)        => { exp!(@sym, options,         SymOptions) };
573    (Out)            => { exp!(@sym, out,             SymOut) };
574    (Pin)            => { exp!(@sym, pin,             SymPin) };
575    (PreservesFlags) => { exp!(@sym, preserves_flags, SymPreservesFlags) };
576    (Pure)           => { exp!(@sym, pure,            SymPure) };
577    (Readonly)       => { exp!(@sym, readonly,        SymReadonly) };
578    (Sym)            => { exp!(@sym, sym,             SymSym) };
579}
580
581/// A bitset type designed specifically for `Parser::expected_token_types`,
582/// which is very hot. `u128` is the smallest integer that will fit every
583/// `TokenType` value.
584#[derive(#[automatically_derived]
impl ::core::clone::Clone for TokenTypeSet {
    #[inline]
    fn clone(&self) -> TokenTypeSet {
        let _: ::core::clone::AssertParamIsClone<u128>;
        *self
    }
}Clone, #[automatically_derived]
impl ::core::marker::Copy for TokenTypeSet { }Copy)]
585pub(super) struct TokenTypeSet(u128);
586
587impl TokenTypeSet {
588    pub(super) const fn new() -> TokenTypeSet {
589        TokenTypeSet(0)
590    }
591
592    pub(super) fn is_empty(&self) -> bool {
593        self.0 == 0
594    }
595
596    pub(super) fn insert(&mut self, token_type: TokenType) {
597        self.0 = self.0 | (1u128 << token_type as u32)
598    }
599
600    pub(super) fn clear(&mut self) {
601        self.0 = 0
602    }
603
604    pub(super) fn contains(&self, token_type: TokenType) -> bool {
605        self.0 & (1u128 << token_type as u32) != 0
606    }
607
608    pub(super) fn iter(&self) -> TokenTypeSetIter {
609        TokenTypeSetIter(*self)
610    }
611}
612
613// The `TokenTypeSet` is a copy of the set being iterated. It initially holds
614// the entire set. Each bit is cleared as it is returned. We have finished once
615// it is all zeroes.
616pub(super) struct TokenTypeSetIter(TokenTypeSet);
617
618impl Iterator for TokenTypeSetIter {
619    type Item = TokenType;
620
621    fn next(&mut self) -> Option<TokenType> {
622        let num_bits: u32 = (size_of_val(&self.0.0) * 8) as u32;
623        match (&num_bits, &128) {
    (left_val, right_val) => {
        if !(*left_val == *right_val) {
            let kind = ::core::panicking::AssertKind::Eq;
            ::core::panicking::assert_failed(kind, &*left_val, &*right_val,
                ::core::option::Option::None);
        }
    }
};assert_eq!(num_bits, 128);
624        let z = self.0.0.trailing_zeros();
625        if z == num_bits {
626            None
627        } else {
628            self.0.0 &= !(1 << z); // clear the trailing 1 bit
629            Some(TokenType::from_u32(z))
630        }
631    }
632}
rustc_parse/parser/token_type.rs

rustc_parse/parser/
token_type.rs
