rustc_parse/parser/
nonterminal.rs

1use std::sync::Arc;
2
3use rustc_ast::HasTokens;
4use rustc_ast::ptr::P;
5use rustc_ast::token::Nonterminal::*;
6use rustc_ast::token::NtExprKind::*;
7use rustc_ast::token::NtPatKind::*;
8use rustc_ast::token::{
9    self, Delimiter, InvisibleOrigin, MetaVarKind, Nonterminal, NonterminalKind, Token,
10};
11use rustc_ast_pretty::pprust;
12use rustc_errors::PResult;
13use rustc_span::{Ident, kw};
14
15use crate::errors::UnexpectedNonterminal;
16use crate::parser::pat::{CommaRecoveryMode, RecoverColon, RecoverComma};
17use crate::parser::{FollowedByType, ForceCollect, ParseNtResult, Parser, PathStyle};
18
19impl<'a> Parser<'a> {
20    /// Checks whether a non-terminal may begin with a particular token.
21    ///
22    /// Returning `false` is a *stability guarantee* that such a matcher will *never* begin with
23    /// that token. Be conservative (return true) if not sure. Inlined because it has a single call
24    /// site.
25    #[inline]
26    pub fn nonterminal_may_begin_with(kind: NonterminalKind, token: &Token) -> bool {
27        /// Checks whether the non-terminal may contain a single (non-keyword) identifier.
28        fn may_be_ident(kind: MetaVarKind) -> bool {
29            match kind {
30                MetaVarKind::Stmt
31                | MetaVarKind::Pat(_)
32                | MetaVarKind::Expr { .. }
33                | MetaVarKind::Ty { .. }
34                | MetaVarKind::Literal // `true`, `false`
35                | MetaVarKind::Meta { .. }
36                | MetaVarKind::Path => true,
37
38                MetaVarKind::Item
39                | MetaVarKind::Block
40                | MetaVarKind::Vis => false,
41
42                MetaVarKind::Ident
43                | MetaVarKind::Lifetime
44                | MetaVarKind::TT => unreachable!(),
45            }
46        }
47
48        /// Old variant of `may_be_ident`. Being phased out.
49        fn nt_may_be_ident(nt: &Nonterminal) -> bool {
50            match nt {
51                NtExpr(_)
52                | NtLiteral(_) // `true`, `false`
53                => true,
54
55                NtBlock(_) => false,
56            }
57        }
58
59        match kind {
60            // `expr_2021` and earlier
61            NonterminalKind::Expr(Expr2021 { .. }) => {
62                token.can_begin_expr()
63                // This exception is here for backwards compatibility.
64                && !token.is_keyword(kw::Let)
65                // This exception is here for backwards compatibility.
66                && !token.is_keyword(kw::Const)
67            }
68            // Current edition expressions
69            NonterminalKind::Expr(Expr) => {
70                // In Edition 2024, `_` is considered an expression, so we
71                // need to allow it here because `token.can_begin_expr()` does
72                // not consider `_` to be an expression.
73                //
74                // Because `can_begin_expr` is used elsewhere, we need to reduce
75                // the scope of where the `_` is considered an expression to
76                // just macro parsing code.
77                (token.can_begin_expr() || token.is_keyword(kw::Underscore))
78                // This exception is here for backwards compatibility.
79                && !token.is_keyword(kw::Let)
80            }
81            NonterminalKind::Ty => token.can_begin_type(),
82            NonterminalKind::Ident => get_macro_ident(token).is_some(),
83            NonterminalKind::Literal => token.can_begin_literal_maybe_minus(),
84            NonterminalKind::Vis => match token.kind {
85                // The follow-set of :vis + "priv" keyword + interpolated/metavar-expansion.
86                token::Comma
87                | token::Ident(..)
88                | token::NtIdent(..)
89                | token::NtLifetime(..)
90                | token::Interpolated(_)
91                | token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(_))) => true,
92                _ => token.can_begin_type(),
93            },
94            NonterminalKind::Block => match &token.kind {
95                token::OpenDelim(Delimiter::Brace) => true,
96                token::NtLifetime(..) => true,
97                token::Interpolated(nt) => match &**nt {
98                    NtBlock(_) | NtExpr(_) | NtLiteral(_) => true,
99                },
100                token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(k))) => match k {
101                    MetaVarKind::Block
102                    | MetaVarKind::Stmt
103                    | MetaVarKind::Expr { .. }
104                    | MetaVarKind::Literal => true,
105                    MetaVarKind::Item
106                    | MetaVarKind::Pat(_)
107                    | MetaVarKind::Ty { .. }
108                    | MetaVarKind::Meta { .. }
109                    | MetaVarKind::Path
110                    | MetaVarKind::Vis => false,
111                    MetaVarKind::Lifetime | MetaVarKind::Ident | MetaVarKind::TT => {
112                        unreachable!()
113                    }
114                },
115                _ => false,
116            },
117            NonterminalKind::Path | NonterminalKind::Meta => match &token.kind {
118                token::PathSep | token::Ident(..) | token::NtIdent(..) => true,
119                token::Interpolated(nt) => nt_may_be_ident(nt),
120                token::OpenDelim(Delimiter::Invisible(InvisibleOrigin::MetaVar(kind))) => {
121                    may_be_ident(*kind)
122                }
123                _ => false,
124            },
125            NonterminalKind::Pat(pat_kind) => token.can_begin_pattern(pat_kind),
126            NonterminalKind::Lifetime => match &token.kind {
127                token::Lifetime(..) | token::NtLifetime(..) => true,
128                _ => false,
129            },
130            NonterminalKind::TT | NonterminalKind::Item | NonterminalKind::Stmt => {
131                !matches!(token.kind, token::CloseDelim(_))
132            }
133        }
134    }
135
136    /// Parse a non-terminal (e.g. MBE `:pat` or `:ident`). Inlined because there is only one call
137    /// site.
138    #[inline]
139    pub fn parse_nonterminal(&mut self, kind: NonterminalKind) -> PResult<'a, ParseNtResult> {
140        // A `macro_rules!` invocation may pass a captured item/expr to a proc-macro,
141        // which requires having captured tokens available. Since we cannot determine
142        // in advance whether or not a proc-macro will be (transitively) invoked,
143        // we always capture tokens for any `Nonterminal` which needs them.
144        let mut nt = match kind {
145            // Note that TT is treated differently to all the others.
146            NonterminalKind::TT => return Ok(ParseNtResult::Tt(self.parse_token_tree())),
147            NonterminalKind::Item => match self.parse_item(ForceCollect::Yes)? {
148                Some(item) => return Ok(ParseNtResult::Item(item)),
149                None => {
150                    return Err(self
151                        .dcx()
152                        .create_err(UnexpectedNonterminal::Item(self.token.span)));
153                }
154            },
155            NonterminalKind::Block => {
156                // While a block *expression* may have attributes (e.g. `#[my_attr] { ... }`),
157                // the ':block' matcher does not support them
158                NtBlock(self.collect_tokens_no_attrs(|this| this.parse_block())?)
159            }
160            NonterminalKind::Stmt => match self.parse_stmt(ForceCollect::Yes)? {
161                Some(stmt) => return Ok(ParseNtResult::Stmt(P(stmt))),
162                None => {
163                    return Err(self
164                        .dcx()
165                        .create_err(UnexpectedNonterminal::Statement(self.token.span)));
166                }
167            },
168            NonterminalKind::Pat(pat_kind) => {
169                return Ok(ParseNtResult::Pat(
170                    self.collect_tokens_no_attrs(|this| match pat_kind {
171                        PatParam { .. } => this.parse_pat_no_top_alt(None, None),
172                        PatWithOr => this.parse_pat_no_top_guard(
173                            None,
174                            RecoverComma::No,
175                            RecoverColon::No,
176                            CommaRecoveryMode::EitherTupleOrPipe,
177                        ),
178                    })?,
179                    pat_kind,
180                ));
181            }
182            NonterminalKind::Expr(_) => NtExpr(self.parse_expr_force_collect()?),
183            NonterminalKind::Literal => {
184                // The `:literal` matcher does not support attributes
185                NtLiteral(self.collect_tokens_no_attrs(|this| this.parse_literal_maybe_minus())?)
186            }
187            NonterminalKind::Ty => {
188                return Ok(ParseNtResult::Ty(
189                    self.collect_tokens_no_attrs(|this| this.parse_ty_no_question_mark_recover())?,
190                ));
191            }
192            // this could be handled like a token, since it is one
193            NonterminalKind::Ident => {
194                return if let Some((ident, is_raw)) = get_macro_ident(&self.token) {
195                    self.bump();
196                    Ok(ParseNtResult::Ident(ident, is_raw))
197                } else {
198                    Err(self.dcx().create_err(UnexpectedNonterminal::Ident {
199                        span: self.token.span,
200                        token: self.token.clone(),
201                    }))
202                };
203            }
204            NonterminalKind::Path => {
205                return Ok(ParseNtResult::Path(P(
206                    self.collect_tokens_no_attrs(|this| this.parse_path(PathStyle::Type))?
207                )));
208            }
209            NonterminalKind::Meta => {
210                return Ok(ParseNtResult::Meta(P(self.parse_attr_item(ForceCollect::Yes)?)));
211            }
212            NonterminalKind::Vis => {
213                return Ok(ParseNtResult::Vis(P(self.collect_tokens_no_attrs(|this| {
214                    this.parse_visibility(FollowedByType::Yes)
215                })?)));
216            }
217            NonterminalKind::Lifetime => {
218                // We want to keep `'keyword` parsing, just like `keyword` is still
219                // an ident for nonterminal purposes.
220                return if let Some((ident, is_raw)) = self.token.lifetime() {
221                    self.bump();
222                    Ok(ParseNtResult::Lifetime(ident, is_raw))
223                } else {
224                    Err(self.dcx().create_err(UnexpectedNonterminal::Lifetime {
225                        span: self.token.span,
226                        token: self.token.clone(),
227                    }))
228                };
229            }
230        };
231
232        // If tokens are supported at all, they should be collected.
233        if matches!(nt.tokens_mut(), Some(None)) {
234            panic!(
235                "Missing tokens for nt {:?} at {:?}: {:?}",
236                nt,
237                nt.use_span(),
238                pprust::nonterminal_to_string(&nt)
239            );
240        }
241
242        Ok(ParseNtResult::Nt(Arc::new(nt)))
243    }
244}
245
246/// The token is an identifier, but not `_`.
247/// We prohibit passing `_` to macros expecting `ident` for now.
248fn get_macro_ident(token: &Token) -> Option<(Ident, token::IdentIsRaw)> {
249    token.ident().filter(|(ident, _)| ident.name != kw::Underscore)
250}
rustc_parse/parser/nonterminal.rs

rustc_parse/parser/
nonterminal.rs
