rustc_ast/
tokenstream.rs

Help
1//! # Token Streams
2//!
3//! `TokenStream`s represent syntactic objects before they are converted into ASTs.
4//! A `TokenStream` is, roughly speaking, a sequence of [`TokenTree`]s,
5//! which are themselves a single [`Token`] or a `Delimited` subsequence of tokens.
6//!
7//! ## Ownership
8//!
9//! `TokenStream`s are persistent data structures constructed as ropes with reference
10//! counted-children. In general, this means that calling an operation on a `TokenStream`
11//! (such as `slice`) produces an entirely new `TokenStream` from the borrowed reference to
12//! the original. This essentially coerces `TokenStream`s into "views" of their subparts,
13//! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking
14//! ownership of the original.
15
16use std::borrow::Cow;
17use std::sync::Arc;
18use std::{cmp, fmt, iter};
19
20use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
21use rustc_data_structures::sync;
22use rustc_macros::{Decodable, Encodable, HashStable_Generic};
23use rustc_serialize::{Decodable, Encodable};
24use rustc_span::{DUMMY_SP, Span, SpanDecoder, SpanEncoder, Symbol, sym};
25
26use crate::ast::{AttrStyle, StmtKind};
27use crate::ast_traits::{HasAttrs, HasTokens};
28use crate::token::{self, Delimiter, InvisibleOrigin, Nonterminal, Token, TokenKind};
29use crate::{AttrVec, Attribute};
30
31/// Part of a `TokenStream`.
32#[derive(Debug, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
33pub enum TokenTree {
34    /// A single token. Should never be `OpenDelim` or `CloseDelim`, because
35    /// delimiters are implicitly represented by `Delimited`.
36    Token(Token, Spacing),
37    /// A delimited sequence of token trees.
38    Delimited(DelimSpan, DelimSpacing, Delimiter, TokenStream),
39}
40
41// Ensure all fields of `TokenTree` are `DynSend` and `DynSync`.
42fn _dummy()
43where
44    Token: sync::DynSend + sync::DynSync,
45    Spacing: sync::DynSend + sync::DynSync,
46    DelimSpan: sync::DynSend + sync::DynSync,
47    Delimiter: sync::DynSend + sync::DynSync,
48    TokenStream: sync::DynSend + sync::DynSync,
49{
50}
51
52impl TokenTree {
53    /// Checks if this `TokenTree` is equal to the other, regardless of span/spacing information.
54    pub fn eq_unspanned(&self, other: &TokenTree) -> bool {
55        match (self, other) {
56            (TokenTree::Token(token, _), TokenTree::Token(token2, _)) => token.kind == token2.kind,
57            (TokenTree::Delimited(.., delim, tts), TokenTree::Delimited(.., delim2, tts2)) => {
58                delim == delim2 && tts.eq_unspanned(tts2)
59            }
60            _ => false,
61        }
62    }
63
64    /// Retrieves the `TokenTree`'s span.
65    pub fn span(&self) -> Span {
66        match self {
67            TokenTree::Token(token, _) => token.span,
68            TokenTree::Delimited(sp, ..) => sp.entire(),
69        }
70    }
71
72    /// Create a `TokenTree::Token` with alone spacing.
73    pub fn token_alone(kind: TokenKind, span: Span) -> TokenTree {
74        TokenTree::Token(Token::new(kind, span), Spacing::Alone)
75    }
76
77    /// Create a `TokenTree::Token` with joint spacing.
78    pub fn token_joint(kind: TokenKind, span: Span) -> TokenTree {
79        TokenTree::Token(Token::new(kind, span), Spacing::Joint)
80    }
81
82    /// Create a `TokenTree::Token` with joint-hidden spacing.
83    pub fn token_joint_hidden(kind: TokenKind, span: Span) -> TokenTree {
84        TokenTree::Token(Token::new(kind, span), Spacing::JointHidden)
85    }
86
87    pub fn uninterpolate(&self) -> Cow<'_, TokenTree> {
88        match self {
89            TokenTree::Token(token, spacing) => match token.uninterpolate() {
90                Cow::Owned(token) => Cow::Owned(TokenTree::Token(token, *spacing)),
91                Cow::Borrowed(_) => Cow::Borrowed(self),
92            },
93            _ => Cow::Borrowed(self),
94        }
95    }
96}
97
98impl<CTX> HashStable<CTX> for TokenStream
99where
100    CTX: crate::HashStableContext,
101{
102    fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
103        for sub_tt in self.iter() {
104            sub_tt.hash_stable(hcx, hasher);
105        }
106    }
107}
108
109pub trait ToAttrTokenStream: sync::DynSend + sync::DynSync {
110    fn to_attr_token_stream(&self) -> AttrTokenStream;
111}
112
113impl ToAttrTokenStream for AttrTokenStream {
114    fn to_attr_token_stream(&self) -> AttrTokenStream {
115        self.clone()
116    }
117}
118
119/// A lazy version of [`TokenStream`], which defers creation
120/// of an actual `TokenStream` until it is needed.
121/// `Box` is here only to reduce the structure size.
122#[derive(Clone)]
123pub struct LazyAttrTokenStream(Arc<Box<dyn ToAttrTokenStream>>);
124
125impl LazyAttrTokenStream {
126    pub fn new(inner: impl ToAttrTokenStream + 'static) -> LazyAttrTokenStream {
127        LazyAttrTokenStream(Arc::new(Box::new(inner)))
128    }
129
130    pub fn to_attr_token_stream(&self) -> AttrTokenStream {
131        self.0.to_attr_token_stream()
132    }
133}
134
135impl fmt::Debug for LazyAttrTokenStream {
136    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
137        write!(f, "LazyAttrTokenStream({:?})", self.to_attr_token_stream())
138    }
139}
140
141impl<S: SpanEncoder> Encodable<S> for LazyAttrTokenStream {
142    fn encode(&self, _s: &mut S) {
143        panic!("Attempted to encode LazyAttrTokenStream");
144    }
145}
146
147impl<D: SpanDecoder> Decodable<D> for LazyAttrTokenStream {
148    fn decode(_d: &mut D) -> Self {
149        panic!("Attempted to decode LazyAttrTokenStream");
150    }
151}
152
153impl<CTX> HashStable<CTX> for LazyAttrTokenStream {
154    fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) {
155        panic!("Attempted to compute stable hash for LazyAttrTokenStream");
156    }
157}
158
159/// An `AttrTokenStream` is similar to a `TokenStream`, but with extra
160/// information about the tokens for attribute targets. This is used
161/// during expansion to perform early cfg-expansion, and to process attributes
162/// during proc-macro invocations.
163#[derive(Clone, Debug, Default, Encodable, Decodable)]
164pub struct AttrTokenStream(pub Arc<Vec<AttrTokenTree>>);
165
166/// Like `TokenTree`, but for `AttrTokenStream`.
167#[derive(Clone, Debug, Encodable, Decodable)]
168pub enum AttrTokenTree {
169    Token(Token, Spacing),
170    Delimited(DelimSpan, DelimSpacing, Delimiter, AttrTokenStream),
171    /// Stores the attributes for an attribute target,
172    /// along with the tokens for that attribute target.
173    /// See `AttrsTarget` for more information
174    AttrsTarget(AttrsTarget),
175}
176
177impl AttrTokenStream {
178    pub fn new(tokens: Vec<AttrTokenTree>) -> AttrTokenStream {
179        AttrTokenStream(Arc::new(tokens))
180    }
181
182    /// Converts this `AttrTokenStream` to a plain `Vec<TokenTree>`. During
183    /// conversion, any `AttrTokenTree::AttrsTarget` gets "flattened" back to a
184    /// `TokenStream`, as described in the comment on
185    /// `attrs_and_tokens_to_token_trees`.
186    pub fn to_token_trees(&self) -> Vec<TokenTree> {
187        let mut res = Vec::with_capacity(self.0.len());
188        for tree in self.0.iter() {
189            match tree {
190                AttrTokenTree::Token(inner, spacing) => {
191                    res.push(TokenTree::Token(inner.clone(), *spacing));
192                }
193                AttrTokenTree::Delimited(span, spacing, delim, stream) => {
194                    res.push(TokenTree::Delimited(
195                        *span,
196                        *spacing,
197                        *delim,
198                        TokenStream::new(stream.to_token_trees()),
199                    ))
200                }
201                AttrTokenTree::AttrsTarget(target) => {
202                    attrs_and_tokens_to_token_trees(&target.attrs, &target.tokens, &mut res);
203                }
204            }
205        }
206        res
207    }
208}
209
210// Converts multiple attributes and the tokens for a target AST node into token trees, and appends
211// them to `res`.
212//
213// Example: if the AST node is "fn f() { blah(); }", then:
214// - Simple if no attributes are present, e.g. "fn f() { blah(); }"
215// - Simple if only outer attribute are present, e.g. "#[outer1] #[outer2] fn f() { blah(); }"
216// - Trickier if inner attributes are present, because they must be moved within the AST node's
217//   tokens, e.g. "#[outer] fn f() { #![inner] blah() }"
218fn attrs_and_tokens_to_token_trees(
219    attrs: &[Attribute],
220    target_tokens: &LazyAttrTokenStream,
221    res: &mut Vec<TokenTree>,
222) {
223    let idx = attrs.partition_point(|attr| matches!(attr.style, crate::AttrStyle::Outer));
224    let (outer_attrs, inner_attrs) = attrs.split_at(idx);
225
226    // Add outer attribute tokens.
227    for attr in outer_attrs {
228        res.extend(attr.token_trees());
229    }
230
231    // Add target AST node tokens.
232    res.extend(target_tokens.to_attr_token_stream().to_token_trees());
233
234    // Insert inner attribute tokens.
235    if !inner_attrs.is_empty() {
236        let mut found = false;
237        // Check the last two trees (to account for a trailing semi)
238        for tree in res.iter_mut().rev().take(2) {
239            if let TokenTree::Delimited(span, spacing, delim, delim_tokens) = tree {
240                // Inner attributes are only supported on extern blocks, functions,
241                // impls, and modules. All of these have their inner attributes
242                // placed at the beginning of the rightmost outermost braced group:
243                // e.g. fn foo() { #![my_attr] }
244                //
245                // Therefore, we can insert them back into the right location
246                // without needing to do any extra position tracking.
247                //
248                // Note: Outline modules are an exception - they can
249                // have attributes like `#![my_attr]` at the start of a file.
250                // Support for custom attributes in this position is not
251                // properly implemented - we always synthesize fake tokens,
252                // so we never reach this code.
253                let mut tts = vec![];
254                for inner_attr in inner_attrs {
255                    tts.extend(inner_attr.token_trees());
256                }
257                tts.extend(delim_tokens.0.iter().cloned());
258                let stream = TokenStream::new(tts);
259                *tree = TokenTree::Delimited(*span, *spacing, *delim, stream);
260                found = true;
261                break;
262            }
263        }
264        assert!(found, "Failed to find trailing delimited group in: {res:?}");
265    }
266}
267
268/// Stores the tokens for an attribute target, along
269/// with its attributes.
270///
271/// This is constructed during parsing when we need to capture
272/// tokens, for `cfg` and `cfg_attr` attributes.
273///
274/// For example, `#[cfg(FALSE)] struct Foo {}` would
275/// have an `attrs` field containing the `#[cfg(FALSE)]` attr,
276/// and a `tokens` field storing the (unparsed) tokens `struct Foo {}`
277///
278/// The `cfg`/`cfg_attr` processing occurs in
279/// `StripUnconfigured::configure_tokens`.
280#[derive(Clone, Debug, Encodable, Decodable)]
281pub struct AttrsTarget {
282    /// Attributes, both outer and inner.
283    /// These are stored in the original order that they were parsed in.
284    pub attrs: AttrVec,
285    /// The underlying tokens for the attribute target that `attrs`
286    /// are applied to
287    pub tokens: LazyAttrTokenStream,
288}
289
290/// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s.
291///
292/// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s
293/// instead of a representation of the abstract syntax tree.
294/// Today's `TokenTree`s can still contain AST via `token::Interpolated` for
295/// backwards compatibility.
296#[derive(Clone, Debug, Default, Encodable, Decodable)]
297pub struct TokenStream(pub(crate) Arc<Vec<TokenTree>>);
298
299/// Indicates whether a token can join with the following token to form a
300/// compound token. Used for conversions to `proc_macro::Spacing`. Also used to
301/// guide pretty-printing, which is where the `JointHidden` value (which isn't
302/// part of `proc_macro::Spacing`) comes in useful.
303#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
304pub enum Spacing {
305    /// The token cannot join with the following token to form a compound
306    /// token.
307    ///
308    /// In token streams parsed from source code, the compiler will use `Alone`
309    /// for any token immediately followed by whitespace, a non-doc comment, or
310    /// EOF.
311    ///
312    /// When constructing token streams within the compiler, use this for each
313    /// token that (a) should be pretty-printed with a space after it, or (b)
314    /// is the last token in the stream. (In the latter case the choice of
315    /// spacing doesn't matter because it is never used for the last token. We
316    /// arbitrarily use `Alone`.)
317    ///
318    /// Converts to `proc_macro::Spacing::Alone`, and
319    /// `proc_macro::Spacing::Alone` converts back to this.
320    Alone,
321
322    /// The token can join with the following token to form a compound token.
323    ///
324    /// In token streams parsed from source code, the compiler will use `Joint`
325    /// for any token immediately followed by punctuation (as determined by
326    /// `Token::is_punct`).
327    ///
328    /// When constructing token streams within the compiler, use this for each
329    /// token that (a) should be pretty-printed without a space after it, and
330    /// (b) is followed by a punctuation token.
331    ///
332    /// Converts to `proc_macro::Spacing::Joint`, and
333    /// `proc_macro::Spacing::Joint` converts back to this.
334    Joint,
335
336    /// The token can join with the following token to form a compound token,
337    /// but this will not be visible at the proc macro level. (This is what the
338    /// `Hidden` means; see below.)
339    ///
340    /// In token streams parsed from source code, the compiler will use
341    /// `JointHidden` for any token immediately followed by anything not
342    /// covered by the `Alone` and `Joint` cases: an identifier, lifetime,
343    /// literal, delimiter, doc comment.
344    ///
345    /// When constructing token streams, use this for each token that (a)
346    /// should be pretty-printed without a space after it, and (b) is followed
347    /// by a non-punctuation token.
348    ///
349    /// Converts to `proc_macro::Spacing::Alone`, but
350    /// `proc_macro::Spacing::Alone` converts back to `token::Spacing::Alone`.
351    /// Because of that, pretty-printing of `TokenStream`s produced by proc
352    /// macros is unavoidably uglier (with more whitespace between tokens) than
353    /// pretty-printing of `TokenStream`'s produced by other means (i.e. parsed
354    /// source code, internally constructed token streams, and token streams
355    /// produced by declarative macros).
356    JointHidden,
357}
358
359impl TokenStream {
360    /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream`
361    /// separating the two arguments with a comma for diagnostic suggestions.
362    pub fn add_comma(&self) -> Option<(TokenStream, Span)> {
363        // Used to suggest if a user writes `foo!(a b);`
364        let mut suggestion = None;
365        let mut iter = self.0.iter().enumerate().peekable();
366        while let Some((pos, ts)) = iter.next() {
367            if let Some((_, next)) = iter.peek() {
368                let sp = match (&ts, &next) {
369                    (_, TokenTree::Token(Token { kind: token::Comma, .. }, _)) => continue,
370                    (
371                        TokenTree::Token(token_left, Spacing::Alone),
372                        TokenTree::Token(token_right, _),
373                    ) if ((token_left.is_ident() && !token_left.is_reserved_ident())
374                        || token_left.is_lit())
375                        && ((token_right.is_ident() && !token_right.is_reserved_ident())
376                            || token_right.is_lit()) =>
377                    {
378                        token_left.span
379                    }
380                    (TokenTree::Delimited(sp, ..), _) => sp.entire(),
381                    _ => continue,
382                };
383                let sp = sp.shrink_to_hi();
384                let comma = TokenTree::token_alone(token::Comma, sp);
385                suggestion = Some((pos, comma, sp));
386            }
387        }
388        if let Some((pos, comma, sp)) = suggestion {
389            let mut new_stream = Vec::with_capacity(self.0.len() + 1);
390            let parts = self.0.split_at(pos + 1);
391            new_stream.extend_from_slice(parts.0);
392            new_stream.push(comma);
393            new_stream.extend_from_slice(parts.1);
394            return Some((TokenStream::new(new_stream), sp));
395        }
396        None
397    }
398}
399
400impl FromIterator<TokenTree> for TokenStream {
401    fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self {
402        TokenStream::new(iter.into_iter().collect::<Vec<TokenTree>>())
403    }
404}
405
406impl Eq for TokenStream {}
407
408impl PartialEq<TokenStream> for TokenStream {
409    fn eq(&self, other: &TokenStream) -> bool {
410        self.iter().eq(other.iter())
411    }
412}
413
414impl TokenStream {
415    pub fn new(tts: Vec<TokenTree>) -> TokenStream {
416        TokenStream(Arc::new(tts))
417    }
418
419    pub fn is_empty(&self) -> bool {
420        self.0.is_empty()
421    }
422
423    pub fn len(&self) -> usize {
424        self.0.len()
425    }
426
427    pub fn get(&self, index: usize) -> Option<&TokenTree> {
428        self.0.get(index)
429    }
430
431    pub fn iter(&self) -> TokenStreamIter<'_> {
432        TokenStreamIter::new(self)
433    }
434
435    /// Compares two `TokenStream`s, checking equality without regarding span information.
436    pub fn eq_unspanned(&self, other: &TokenStream) -> bool {
437        let mut iter1 = self.iter();
438        let mut iter2 = other.iter();
439        for (tt1, tt2) in iter::zip(&mut iter1, &mut iter2) {
440            if !tt1.eq_unspanned(tt2) {
441                return false;
442            }
443        }
444        iter1.next().is_none() && iter2.next().is_none()
445    }
446
447    /// Create a token stream containing a single token with alone spacing. The
448    /// spacing used for the final token in a constructed stream doesn't matter
449    /// because it's never used. In practice we arbitrarily use
450    /// `Spacing::Alone`.
451    pub fn token_alone(kind: TokenKind, span: Span) -> TokenStream {
452        TokenStream::new(vec![TokenTree::token_alone(kind, span)])
453    }
454
455    pub fn from_ast(node: &(impl HasAttrs + HasTokens + fmt::Debug)) -> TokenStream {
456        let tokens = node.tokens().unwrap_or_else(|| panic!("missing tokens for node: {:?}", node));
457        let mut tts = vec![];
458        attrs_and_tokens_to_token_trees(node.attrs(), tokens, &mut tts);
459        TokenStream::new(tts)
460    }
461
462    pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
463        match nt {
464            Nonterminal::NtItem(item) => TokenStream::from_ast(item),
465            Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
466            Nonterminal::NtStmt(stmt) if let StmtKind::Empty = stmt.kind => {
467                // FIXME: Properly collect tokens for empty statements.
468                TokenStream::token_alone(token::Semi, stmt.span)
469            }
470            Nonterminal::NtStmt(stmt) => TokenStream::from_ast(stmt),
471            Nonterminal::NtPat(pat) => TokenStream::from_ast(pat),
472            Nonterminal::NtTy(ty) => TokenStream::from_ast(ty),
473            Nonterminal::NtMeta(attr) => TokenStream::from_ast(attr),
474            Nonterminal::NtPath(path) => TokenStream::from_ast(path),
475            Nonterminal::NtVis(vis) => TokenStream::from_ast(vis),
476            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),
477        }
478    }
479
480    fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree {
481        match token.kind {
482            token::NtIdent(ident, is_raw) => {
483                TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing)
484            }
485            token::NtLifetime(ident, is_raw) => TokenTree::Delimited(
486                DelimSpan::from_single(token.span),
487                DelimSpacing::new(Spacing::JointHidden, spacing),
488                Delimiter::Invisible(InvisibleOrigin::FlattenToken),
489                TokenStream::token_alone(token::Lifetime(ident.name, is_raw), ident.span),
490            ),
491            token::Interpolated(ref nt) => TokenTree::Delimited(
492                DelimSpan::from_single(token.span),
493                DelimSpacing::new(Spacing::JointHidden, spacing),
494                Delimiter::Invisible(InvisibleOrigin::FlattenToken),
495                TokenStream::from_nonterminal_ast(&nt).flattened(),
496            ),
497            _ => TokenTree::Token(token.clone(), spacing),
498        }
499    }
500
501    fn flatten_token_tree(tree: &TokenTree) -> TokenTree {
502        match tree {
503            TokenTree::Token(token, spacing) => TokenStream::flatten_token(token, *spacing),
504            TokenTree::Delimited(span, spacing, delim, tts) => {
505                TokenTree::Delimited(*span, *spacing, *delim, tts.flattened())
506            }
507        }
508    }
509
510    #[must_use]
511    pub fn flattened(&self) -> TokenStream {
512        fn can_skip(stream: &TokenStream) -> bool {
513            stream.iter().all(|tree| match tree {
514                TokenTree::Token(token, _) => !matches!(
515                    token.kind,
516                    token::NtIdent(..) | token::NtLifetime(..) | token::Interpolated(..)
517                ),
518                TokenTree::Delimited(.., inner) => can_skip(inner),
519            })
520        }
521
522        if can_skip(self) {
523            return self.clone();
524        }
525
526        self.iter().map(|tree| TokenStream::flatten_token_tree(tree)).collect()
527    }
528
529    // If `vec` is not empty, try to glue `tt` onto its last token. The return
530    // value indicates if gluing took place.
531    fn try_glue_to_last(vec: &mut Vec<TokenTree>, tt: &TokenTree) -> bool {
532        if let Some(TokenTree::Token(last_tok, Spacing::Joint | Spacing::JointHidden)) = vec.last()
533            && let TokenTree::Token(tok, spacing) = tt
534            && let Some(glued_tok) = last_tok.glue(tok)
535        {
536            // ...then overwrite the last token tree in `vec` with the
537            // glued token, and skip the first token tree from `stream`.
538            *vec.last_mut().unwrap() = TokenTree::Token(glued_tok, *spacing);
539            true
540        } else {
541            false
542        }
543    }
544
545    /// Push `tt` onto the end of the stream, possibly gluing it to the last
546    /// token. Uses `make_mut` to maximize efficiency.
547    pub fn push_tree(&mut self, tt: TokenTree) {
548        let vec_mut = Arc::make_mut(&mut self.0);
549
550        if Self::try_glue_to_last(vec_mut, &tt) {
551            // nothing else to do
552        } else {
553            vec_mut.push(tt);
554        }
555    }
556
557    /// Push `stream` onto the end of the stream, possibly gluing the first
558    /// token tree to the last token. (No other token trees will be glued.)
559    /// Uses `make_mut` to maximize efficiency.
560    pub fn push_stream(&mut self, stream: TokenStream) {
561        let vec_mut = Arc::make_mut(&mut self.0);
562
563        let stream_iter = stream.0.iter().cloned();
564
565        if let Some(first) = stream.0.first()
566            && Self::try_glue_to_last(vec_mut, first)
567        {
568            // Now skip the first token tree from `stream`.
569            vec_mut.extend(stream_iter.skip(1));
570        } else {
571            // Append all of `stream`.
572            vec_mut.extend(stream_iter);
573        }
574    }
575
576    pub fn chunks(&self, chunk_size: usize) -> core::slice::Chunks<'_, TokenTree> {
577        self.0.chunks(chunk_size)
578    }
579
580    /// Desugar doc comments like `/// foo` in the stream into `#[doc =
581    /// r"foo"]`. Modifies the `TokenStream` via `Arc::make_mut`, but as little
582    /// as possible.
583    pub fn desugar_doc_comments(&mut self) {
584        if let Some(desugared_stream) = desugar_inner(self.clone()) {
585            *self = desugared_stream;
586        }
587
588        // The return value is `None` if nothing in `stream` changed.
589        fn desugar_inner(mut stream: TokenStream) -> Option<TokenStream> {
590            let mut i = 0;
591            let mut modified = false;
592            while let Some(tt) = stream.0.get(i) {
593                match tt {
594                    &TokenTree::Token(
595                        Token { kind: token::DocComment(_, attr_style, data), span },
596                        _spacing,
597                    ) => {
598                        let desugared = desugared_tts(attr_style, data, span);
599                        let desugared_len = desugared.len();
600                        Arc::make_mut(&mut stream.0).splice(i..i + 1, desugared);
601                        modified = true;
602                        i += desugared_len;
603                    }
604
605                    &TokenTree::Token(..) => i += 1,
606
607                    &TokenTree::Delimited(sp, spacing, delim, ref delim_stream) => {
608                        if let Some(desugared_delim_stream) = desugar_inner(delim_stream.clone()) {
609                            let new_tt =
610                                TokenTree::Delimited(sp, spacing, delim, desugared_delim_stream);
611                            Arc::make_mut(&mut stream.0)[i] = new_tt;
612                            modified = true;
613                        }
614                        i += 1;
615                    }
616                }
617            }
618            if modified { Some(stream) } else { None }
619        }
620
621        fn desugared_tts(attr_style: AttrStyle, data: Symbol, span: Span) -> Vec<TokenTree> {
622            // Searches for the occurrences of `"#*` and returns the minimum number of `#`s
623            // required to wrap the text. E.g.
624            // - `abc d` is wrapped as `r"abc d"` (num_of_hashes = 0)
625            // - `abc "d"` is wrapped as `r#"abc "d""#` (num_of_hashes = 1)
626            // - `abc "##d##"` is wrapped as `r###"abc ##"d"##"###` (num_of_hashes = 3)
627            let mut num_of_hashes = 0;
628            let mut count = 0;
629            for ch in data.as_str().chars() {
630                count = match ch {
631                    '"' => 1,
632                    '#' if count > 0 => count + 1,
633                    _ => 0,
634                };
635                num_of_hashes = cmp::max(num_of_hashes, count);
636            }
637
638            // `/// foo` becomes `[doc = r"foo"]`.
639            let delim_span = DelimSpan::from_single(span);
640            let body = TokenTree::Delimited(
641                delim_span,
642                DelimSpacing::new(Spacing::JointHidden, Spacing::Alone),
643                Delimiter::Bracket,
644                [
645                    TokenTree::token_alone(token::Ident(sym::doc, token::IdentIsRaw::No), span),
646                    TokenTree::token_alone(token::Eq, span),
647                    TokenTree::token_alone(
648                        TokenKind::lit(token::StrRaw(num_of_hashes), data, None),
649                        span,
650                    ),
651                ]
652                .into_iter()
653                .collect::<TokenStream>(),
654            );
655
656            if attr_style == AttrStyle::Inner {
657                vec![
658                    TokenTree::token_joint(token::Pound, span),
659                    TokenTree::token_joint_hidden(token::Not, span),
660                    body,
661                ]
662            } else {
663                vec![TokenTree::token_joint_hidden(token::Pound, span), body]
664            }
665        }
666    }
667}
668
669#[derive(Clone)]
670pub struct TokenStreamIter<'t> {
671    stream: &'t TokenStream,
672    index: usize,
673}
674
675impl<'t> TokenStreamIter<'t> {
676    fn new(stream: &'t TokenStream) -> Self {
677        TokenStreamIter { stream, index: 0 }
678    }
679
680    // Peeking could be done via `Peekable`, but most iterators need peeking,
681    // and this is simple and avoids the need to use `peekable` and `Peekable`
682    // at all the use sites.
683    pub fn peek(&self) -> Option<&'t TokenTree> {
684        self.stream.0.get(self.index)
685    }
686}
687
688impl<'t> Iterator for TokenStreamIter<'t> {
689    type Item = &'t TokenTree;
690
691    fn next(&mut self) -> Option<&'t TokenTree> {
692        self.stream.0.get(self.index).map(|tree| {
693            self.index += 1;
694            tree
695        })
696    }
697}
698
699#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
700pub struct DelimSpan {
701    pub open: Span,
702    pub close: Span,
703}
704
705impl DelimSpan {
706    pub fn from_single(sp: Span) -> Self {
707        DelimSpan { open: sp, close: sp }
708    }
709
710    pub fn from_pair(open: Span, close: Span) -> Self {
711        DelimSpan { open, close }
712    }
713
714    pub fn dummy() -> Self {
715        Self::from_single(DUMMY_SP)
716    }
717
718    pub fn entire(self) -> Span {
719        self.open.with_hi(self.close.hi())
720    }
721}
722
723#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
724pub struct DelimSpacing {
725    pub open: Spacing,
726    pub close: Spacing,
727}
728
729impl DelimSpacing {
730    pub fn new(open: Spacing, close: Spacing) -> DelimSpacing {
731        DelimSpacing { open, close }
732    }
733}
734
735// Some types are used a lot. Make sure they don't unintentionally get bigger.
736#[cfg(target_pointer_width = "64")]
737mod size_asserts {
738    use rustc_data_structures::static_assert_size;
739
740    use super::*;
741    // tidy-alphabetical-start
742    static_assert_size!(AttrTokenStream, 8);
743    static_assert_size!(AttrTokenTree, 32);
744    static_assert_size!(LazyAttrTokenStream, 8);
745    static_assert_size!(Option<LazyAttrTokenStream>, 8); // must be small, used in many AST nodes
746    static_assert_size!(TokenStream, 8);
747    static_assert_size!(TokenTree, 32);
748    // tidy-alphabetical-end
749}
rustc_ast/tokenstream.rs

rustc_ast/
tokenstream.rs
