rustc_ast/
tokenstream.rs

Help
1//! # Token Streams
2//!
3//! `TokenStream`s represent syntactic objects before they are converted into ASTs.
4//! A `TokenStream` is, roughly speaking, a sequence of [`TokenTree`]s,
5//! which are themselves a single [`Token`] or a `Delimited` subsequence of tokens.
6//!
7//! ## Ownership
8//!
9//! `TokenStream`s are persistent data structures constructed as ropes with reference
10//! counted-children. In general, this means that calling an operation on a `TokenStream`
11//! (such as `slice`) produces an entirely new `TokenStream` from the borrowed reference to
12//! the original. This essentially coerces `TokenStream`s into "views" of their subparts,
13//! and a borrowed `TokenStream` is sufficient to build an owned `TokenStream` without taking
14//! ownership of the original.
15
16use std::borrow::Cow;
17use std::sync::Arc;
18use std::{cmp, fmt, iter};
19
20use rustc_data_structures::stable_hasher::{HashStable, StableHasher};
21use rustc_data_structures::sync;
22use rustc_macros::{Decodable, Encodable, HashStable_Generic};
23use rustc_serialize::{Decodable, Encodable};
24use rustc_span::{DUMMY_SP, Span, SpanDecoder, SpanEncoder, Symbol, sym};
25
26use crate::ast::AttrStyle;
27use crate::ast_traits::{HasAttrs, HasTokens};
28use crate::token::{self, Delimiter, InvisibleOrigin, Nonterminal, Token, TokenKind};
29use crate::{AttrVec, Attribute};
30
31/// Part of a `TokenStream`.
32#[derive(Debug, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
33pub enum TokenTree {
34    /// A single token. Should never be `OpenDelim` or `CloseDelim`, because
35    /// delimiters are implicitly represented by `Delimited`.
36    Token(Token, Spacing),
37    /// A delimited sequence of token trees.
38    Delimited(DelimSpan, DelimSpacing, Delimiter, TokenStream),
39}
40
41// Ensure all fields of `TokenTree` are `DynSend` and `DynSync`.
42fn _dummy()
43where
44    Token: sync::DynSend + sync::DynSync,
45    Spacing: sync::DynSend + sync::DynSync,
46    DelimSpan: sync::DynSend + sync::DynSync,
47    Delimiter: sync::DynSend + sync::DynSync,
48    TokenStream: sync::DynSend + sync::DynSync,
49{
50}
51
52impl TokenTree {
53    /// Checks if this `TokenTree` is equal to the other, regardless of span/spacing information.
54    pub fn eq_unspanned(&self, other: &TokenTree) -> bool {
55        match (self, other) {
56            (TokenTree::Token(token, _), TokenTree::Token(token2, _)) => token.kind == token2.kind,
57            (TokenTree::Delimited(.., delim, tts), TokenTree::Delimited(.., delim2, tts2)) => {
58                delim == delim2 && tts.eq_unspanned(tts2)
59            }
60            _ => false,
61        }
62    }
63
64    /// Retrieves the `TokenTree`'s span.
65    pub fn span(&self) -> Span {
66        match self {
67            TokenTree::Token(token, _) => token.span,
68            TokenTree::Delimited(sp, ..) => sp.entire(),
69        }
70    }
71
72    /// Create a `TokenTree::Token` with alone spacing.
73    pub fn token_alone(kind: TokenKind, span: Span) -> TokenTree {
74        TokenTree::Token(Token::new(kind, span), Spacing::Alone)
75    }
76
77    /// Create a `TokenTree::Token` with joint spacing.
78    pub fn token_joint(kind: TokenKind, span: Span) -> TokenTree {
79        TokenTree::Token(Token::new(kind, span), Spacing::Joint)
80    }
81
82    /// Create a `TokenTree::Token` with joint-hidden spacing.
83    pub fn token_joint_hidden(kind: TokenKind, span: Span) -> TokenTree {
84        TokenTree::Token(Token::new(kind, span), Spacing::JointHidden)
85    }
86
87    pub fn uninterpolate(&self) -> Cow<'_, TokenTree> {
88        match self {
89            TokenTree::Token(token, spacing) => match token.uninterpolate() {
90                Cow::Owned(token) => Cow::Owned(TokenTree::Token(token, *spacing)),
91                Cow::Borrowed(_) => Cow::Borrowed(self),
92            },
93            _ => Cow::Borrowed(self),
94        }
95    }
96}
97
98impl<CTX> HashStable<CTX> for TokenStream
99where
100    CTX: crate::HashStableContext,
101{
102    fn hash_stable(&self, hcx: &mut CTX, hasher: &mut StableHasher) {
103        for sub_tt in self.iter() {
104            sub_tt.hash_stable(hcx, hasher);
105        }
106    }
107}
108
109pub trait ToAttrTokenStream: sync::DynSend + sync::DynSync {
110    fn to_attr_token_stream(&self) -> AttrTokenStream;
111}
112
113impl ToAttrTokenStream for AttrTokenStream {
114    fn to_attr_token_stream(&self) -> AttrTokenStream {
115        self.clone()
116    }
117}
118
119/// A lazy version of [`TokenStream`], which defers creation
120/// of an actual `TokenStream` until it is needed.
121/// `Box` is here only to reduce the structure size.
122#[derive(Clone)]
123pub struct LazyAttrTokenStream(Arc<Box<dyn ToAttrTokenStream>>);
124
125impl LazyAttrTokenStream {
126    pub fn new(inner: impl ToAttrTokenStream + 'static) -> LazyAttrTokenStream {
127        LazyAttrTokenStream(Arc::new(Box::new(inner)))
128    }
129
130    pub fn to_attr_token_stream(&self) -> AttrTokenStream {
131        self.0.to_attr_token_stream()
132    }
133}
134
135impl fmt::Debug for LazyAttrTokenStream {
136    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
137        write!(f, "LazyAttrTokenStream({:?})", self.to_attr_token_stream())
138    }
139}
140
141impl<S: SpanEncoder> Encodable<S> for LazyAttrTokenStream {
142    fn encode(&self, _s: &mut S) {
143        panic!("Attempted to encode LazyAttrTokenStream");
144    }
145}
146
147impl<D: SpanDecoder> Decodable<D> for LazyAttrTokenStream {
148    fn decode(_d: &mut D) -> Self {
149        panic!("Attempted to decode LazyAttrTokenStream");
150    }
151}
152
153impl<CTX> HashStable<CTX> for LazyAttrTokenStream {
154    fn hash_stable(&self, _hcx: &mut CTX, _hasher: &mut StableHasher) {
155        panic!("Attempted to compute stable hash for LazyAttrTokenStream");
156    }
157}
158
159/// An `AttrTokenStream` is similar to a `TokenStream`, but with extra
160/// information about the tokens for attribute targets. This is used
161/// during expansion to perform early cfg-expansion, and to process attributes
162/// during proc-macro invocations.
163#[derive(Clone, Debug, Default, Encodable, Decodable)]
164pub struct AttrTokenStream(pub Arc<Vec<AttrTokenTree>>);
165
166/// Like `TokenTree`, but for `AttrTokenStream`.
167#[derive(Clone, Debug, Encodable, Decodable)]
168pub enum AttrTokenTree {
169    Token(Token, Spacing),
170    Delimited(DelimSpan, DelimSpacing, Delimiter, AttrTokenStream),
171    /// Stores the attributes for an attribute target,
172    /// along with the tokens for that attribute target.
173    /// See `AttrsTarget` for more information
174    AttrsTarget(AttrsTarget),
175}
176
177impl AttrTokenStream {
178    pub fn new(tokens: Vec<AttrTokenTree>) -> AttrTokenStream {
179        AttrTokenStream(Arc::new(tokens))
180    }
181
182    /// Converts this `AttrTokenStream` to a plain `Vec<TokenTree>`. During
183    /// conversion, any `AttrTokenTree::AttrsTarget` gets "flattened" back to a
184    /// `TokenStream`, as described in the comment on
185    /// `attrs_and_tokens_to_token_trees`.
186    pub fn to_token_trees(&self) -> Vec<TokenTree> {
187        let mut res = Vec::with_capacity(self.0.len());
188        for tree in self.0.iter() {
189            match tree {
190                AttrTokenTree::Token(inner, spacing) => {
191                    res.push(TokenTree::Token(inner.clone(), *spacing));
192                }
193                AttrTokenTree::Delimited(span, spacing, delim, stream) => {
194                    res.push(TokenTree::Delimited(
195                        *span,
196                        *spacing,
197                        *delim,
198                        TokenStream::new(stream.to_token_trees()),
199                    ))
200                }
201                AttrTokenTree::AttrsTarget(target) => {
202                    attrs_and_tokens_to_token_trees(&target.attrs, &target.tokens, &mut res);
203                }
204            }
205        }
206        res
207    }
208}
209
210// Converts multiple attributes and the tokens for a target AST node into token trees, and appends
211// them to `res`.
212//
213// Example: if the AST node is "fn f() { blah(); }", then:
214// - Simple if no attributes are present, e.g. "fn f() { blah(); }"
215// - Simple if only outer attribute are present, e.g. "#[outer1] #[outer2] fn f() { blah(); }"
216// - Trickier if inner attributes are present, because they must be moved within the AST node's
217//   tokens, e.g. "#[outer] fn f() { #![inner] blah() }"
218fn attrs_and_tokens_to_token_trees(
219    attrs: &[Attribute],
220    target_tokens: &LazyAttrTokenStream,
221    res: &mut Vec<TokenTree>,
222) {
223    let idx = attrs.partition_point(|attr| matches!(attr.style, crate::AttrStyle::Outer));
224    let (outer_attrs, inner_attrs) = attrs.split_at(idx);
225
226    // Add outer attribute tokens.
227    for attr in outer_attrs {
228        res.extend(attr.token_trees());
229    }
230
231    // Add target AST node tokens.
232    res.extend(target_tokens.to_attr_token_stream().to_token_trees());
233
234    // Insert inner attribute tokens.
235    if !inner_attrs.is_empty() {
236        let mut found = false;
237        // Check the last two trees (to account for a trailing semi)
238        for tree in res.iter_mut().rev().take(2) {
239            if let TokenTree::Delimited(span, spacing, delim, delim_tokens) = tree {
240                // Inner attributes are only supported on extern blocks, functions,
241                // impls, and modules. All of these have their inner attributes
242                // placed at the beginning of the rightmost outermost braced group:
243                // e.g. fn foo() { #![my_attr] }
244                //
245                // Therefore, we can insert them back into the right location
246                // without needing to do any extra position tracking.
247                //
248                // Note: Outline modules are an exception - they can
249                // have attributes like `#![my_attr]` at the start of a file.
250                // Support for custom attributes in this position is not
251                // properly implemented - we always synthesize fake tokens,
252                // so we never reach this code.
253                let mut tts = vec![];
254                for inner_attr in inner_attrs {
255                    tts.extend(inner_attr.token_trees());
256                }
257                tts.extend(delim_tokens.0.iter().cloned());
258                let stream = TokenStream::new(tts);
259                *tree = TokenTree::Delimited(*span, *spacing, *delim, stream);
260                found = true;
261                break;
262            }
263        }
264        assert!(found, "Failed to find trailing delimited group in: {res:?}");
265    }
266}
267
268/// Stores the tokens for an attribute target, along
269/// with its attributes.
270///
271/// This is constructed during parsing when we need to capture
272/// tokens, for `cfg` and `cfg_attr` attributes.
273///
274/// For example, `#[cfg(FALSE)] struct Foo {}` would
275/// have an `attrs` field containing the `#[cfg(FALSE)]` attr,
276/// and a `tokens` field storing the (unparsed) tokens `struct Foo {}`
277///
278/// The `cfg`/`cfg_attr` processing occurs in
279/// `StripUnconfigured::configure_tokens`.
280#[derive(Clone, Debug, Encodable, Decodable)]
281pub struct AttrsTarget {
282    /// Attributes, both outer and inner.
283    /// These are stored in the original order that they were parsed in.
284    pub attrs: AttrVec,
285    /// The underlying tokens for the attribute target that `attrs`
286    /// are applied to
287    pub tokens: LazyAttrTokenStream,
288}
289
290/// A `TokenStream` is an abstract sequence of tokens, organized into [`TokenTree`]s.
291///
292/// The goal is for procedural macros to work with `TokenStream`s and `TokenTree`s
293/// instead of a representation of the abstract syntax tree.
294/// Today's `TokenTree`s can still contain AST via `token::Interpolated` for
295/// backwards compatibility.
296#[derive(Clone, Debug, Default, Encodable, Decodable)]
297pub struct TokenStream(pub(crate) Arc<Vec<TokenTree>>);
298
299/// Indicates whether a token can join with the following token to form a
300/// compound token. Used for conversions to `proc_macro::Spacing`. Also used to
301/// guide pretty-printing, which is where the `JointHidden` value (which isn't
302/// part of `proc_macro::Spacing`) comes in useful.
303#[derive(Clone, Copy, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
304pub enum Spacing {
305    /// The token cannot join with the following token to form a compound
306    /// token.
307    ///
308    /// In token streams parsed from source code, the compiler will use `Alone`
309    /// for any token immediately followed by whitespace, a non-doc comment, or
310    /// EOF.
311    ///
312    /// When constructing token streams within the compiler, use this for each
313    /// token that (a) should be pretty-printed with a space after it, or (b)
314    /// is the last token in the stream. (In the latter case the choice of
315    /// spacing doesn't matter because it is never used for the last token. We
316    /// arbitrarily use `Alone`.)
317    ///
318    /// Converts to `proc_macro::Spacing::Alone`, and
319    /// `proc_macro::Spacing::Alone` converts back to this.
320    Alone,
321
322    /// The token can join with the following token to form a compound token.
323    ///
324    /// In token streams parsed from source code, the compiler will use `Joint`
325    /// for any token immediately followed by punctuation (as determined by
326    /// `Token::is_punct`).
327    ///
328    /// When constructing token streams within the compiler, use this for each
329    /// token that (a) should be pretty-printed without a space after it, and
330    /// (b) is followed by a punctuation token.
331    ///
332    /// Converts to `proc_macro::Spacing::Joint`, and
333    /// `proc_macro::Spacing::Joint` converts back to this.
334    Joint,
335
336    /// The token can join with the following token to form a compound token,
337    /// but this will not be visible at the proc macro level. (This is what the
338    /// `Hidden` means; see below.)
339    ///
340    /// In token streams parsed from source code, the compiler will use
341    /// `JointHidden` for any token immediately followed by anything not
342    /// covered by the `Alone` and `Joint` cases: an identifier, lifetime,
343    /// literal, delimiter, doc comment.
344    ///
345    /// When constructing token streams, use this for each token that (a)
346    /// should be pretty-printed without a space after it, and (b) is followed
347    /// by a non-punctuation token.
348    ///
349    /// Converts to `proc_macro::Spacing::Alone`, but
350    /// `proc_macro::Spacing::Alone` converts back to `token::Spacing::Alone`.
351    /// Because of that, pretty-printing of `TokenStream`s produced by proc
352    /// macros is unavoidably uglier (with more whitespace between tokens) than
353    /// pretty-printing of `TokenStream`'s produced by other means (i.e. parsed
354    /// source code, internally constructed token streams, and token streams
355    /// produced by declarative macros).
356    JointHidden,
357}
358
359impl TokenStream {
360    /// Given a `TokenStream` with a `Stream` of only two arguments, return a new `TokenStream`
361    /// separating the two arguments with a comma for diagnostic suggestions.
362    pub fn add_comma(&self) -> Option<(TokenStream, Span)> {
363        // Used to suggest if a user writes `foo!(a b);`
364        let mut suggestion = None;
365        let mut iter = self.0.iter().enumerate().peekable();
366        while let Some((pos, ts)) = iter.next() {
367            if let Some((_, next)) = iter.peek() {
368                let sp = match (&ts, &next) {
369                    (_, TokenTree::Token(Token { kind: token::Comma, .. }, _)) => continue,
370                    (
371                        TokenTree::Token(token_left, Spacing::Alone),
372                        TokenTree::Token(token_right, _),
373                    ) if ((token_left.is_ident() && !token_left.is_reserved_ident())
374                        || token_left.is_lit())
375                        && ((token_right.is_ident() && !token_right.is_reserved_ident())
376                            || token_right.is_lit()) =>
377                    {
378                        token_left.span
379                    }
380                    (TokenTree::Delimited(sp, ..), _) => sp.entire(),
381                    _ => continue,
382                };
383                let sp = sp.shrink_to_hi();
384                let comma = TokenTree::token_alone(token::Comma, sp);
385                suggestion = Some((pos, comma, sp));
386            }
387        }
388        if let Some((pos, comma, sp)) = suggestion {
389            let mut new_stream = Vec::with_capacity(self.0.len() + 1);
390            let parts = self.0.split_at(pos + 1);
391            new_stream.extend_from_slice(parts.0);
392            new_stream.push(comma);
393            new_stream.extend_from_slice(parts.1);
394            return Some((TokenStream::new(new_stream), sp));
395        }
396        None
397    }
398}
399
400impl FromIterator<TokenTree> for TokenStream {
401    fn from_iter<I: IntoIterator<Item = TokenTree>>(iter: I) -> Self {
402        TokenStream::new(iter.into_iter().collect::<Vec<TokenTree>>())
403    }
404}
405
406impl Eq for TokenStream {}
407
408impl PartialEq<TokenStream> for TokenStream {
409    fn eq(&self, other: &TokenStream) -> bool {
410        self.iter().eq(other.iter())
411    }
412}
413
414impl TokenStream {
415    pub fn new(tts: Vec<TokenTree>) -> TokenStream {
416        TokenStream(Arc::new(tts))
417    }
418
419    pub fn is_empty(&self) -> bool {
420        self.0.is_empty()
421    }
422
423    pub fn len(&self) -> usize {
424        self.0.len()
425    }
426
427    pub fn get(&self, index: usize) -> Option<&TokenTree> {
428        self.0.get(index)
429    }
430
431    pub fn iter(&self) -> TokenStreamIter<'_> {
432        TokenStreamIter::new(self)
433    }
434
435    /// Compares two `TokenStream`s, checking equality without regarding span information.
436    pub fn eq_unspanned(&self, other: &TokenStream) -> bool {
437        let mut iter1 = self.iter();
438        let mut iter2 = other.iter();
439        for (tt1, tt2) in iter::zip(&mut iter1, &mut iter2) {
440            if !tt1.eq_unspanned(tt2) {
441                return false;
442            }
443        }
444        iter1.next().is_none() && iter2.next().is_none()
445    }
446
447    /// Create a token stream containing a single token with alone spacing. The
448    /// spacing used for the final token in a constructed stream doesn't matter
449    /// because it's never used. In practice we arbitrarily use
450    /// `Spacing::Alone`.
451    pub fn token_alone(kind: TokenKind, span: Span) -> TokenStream {
452        TokenStream::new(vec![TokenTree::token_alone(kind, span)])
453    }
454
455    pub fn from_ast(node: &(impl HasAttrs + HasTokens + fmt::Debug)) -> TokenStream {
456        let tokens = node.tokens().unwrap_or_else(|| panic!("missing tokens for node: {:?}", node));
457        let mut tts = vec![];
458        attrs_and_tokens_to_token_trees(node.attrs(), tokens, &mut tts);
459        TokenStream::new(tts)
460    }
461
462    pub fn from_nonterminal_ast(nt: &Nonterminal) -> TokenStream {
463        match nt {
464            Nonterminal::NtBlock(block) => TokenStream::from_ast(block),
465            Nonterminal::NtExpr(expr) | Nonterminal::NtLiteral(expr) => TokenStream::from_ast(expr),
466        }
467    }
468
469    fn flatten_token(token: &Token, spacing: Spacing) -> TokenTree {
470        match token.kind {
471            token::NtIdent(ident, is_raw) => {
472                TokenTree::Token(Token::new(token::Ident(ident.name, is_raw), ident.span), spacing)
473            }
474            token::NtLifetime(ident, is_raw) => TokenTree::Delimited(
475                DelimSpan::from_single(token.span),
476                DelimSpacing::new(Spacing::JointHidden, spacing),
477                Delimiter::Invisible(InvisibleOrigin::FlattenToken),
478                TokenStream::token_alone(token::Lifetime(ident.name, is_raw), ident.span),
479            ),
480            token::Interpolated(ref nt) => TokenTree::Delimited(
481                DelimSpan::from_single(token.span),
482                DelimSpacing::new(Spacing::JointHidden, spacing),
483                Delimiter::Invisible(InvisibleOrigin::FlattenToken),
484                TokenStream::from_nonterminal_ast(&nt).flattened(),
485            ),
486            _ => TokenTree::Token(token.clone(), spacing),
487        }
488    }
489
490    fn flatten_token_tree(tree: &TokenTree) -> TokenTree {
491        match tree {
492            TokenTree::Token(token, spacing) => TokenStream::flatten_token(token, *spacing),
493            TokenTree::Delimited(span, spacing, delim, tts) => {
494                TokenTree::Delimited(*span, *spacing, *delim, tts.flattened())
495            }
496        }
497    }
498
499    #[must_use]
500    pub fn flattened(&self) -> TokenStream {
501        fn can_skip(stream: &TokenStream) -> bool {
502            stream.iter().all(|tree| match tree {
503                TokenTree::Token(token, _) => !matches!(
504                    token.kind,
505                    token::NtIdent(..) | token::NtLifetime(..) | token::Interpolated(..)
506                ),
507                TokenTree::Delimited(.., inner) => can_skip(inner),
508            })
509        }
510
511        if can_skip(self) {
512            return self.clone();
513        }
514
515        self.iter().map(|tree| TokenStream::flatten_token_tree(tree)).collect()
516    }
517
518    // If `vec` is not empty, try to glue `tt` onto its last token. The return
519    // value indicates if gluing took place.
520    fn try_glue_to_last(vec: &mut Vec<TokenTree>, tt: &TokenTree) -> bool {
521        if let Some(TokenTree::Token(last_tok, Spacing::Joint | Spacing::JointHidden)) = vec.last()
522            && let TokenTree::Token(tok, spacing) = tt
523            && let Some(glued_tok) = last_tok.glue(tok)
524        {
525            // ...then overwrite the last token tree in `vec` with the
526            // glued token, and skip the first token tree from `stream`.
527            *vec.last_mut().unwrap() = TokenTree::Token(glued_tok, *spacing);
528            true
529        } else {
530            false
531        }
532    }
533
534    /// Push `tt` onto the end of the stream, possibly gluing it to the last
535    /// token. Uses `make_mut` to maximize efficiency.
536    pub fn push_tree(&mut self, tt: TokenTree) {
537        let vec_mut = Arc::make_mut(&mut self.0);
538
539        if Self::try_glue_to_last(vec_mut, &tt) {
540            // nothing else to do
541        } else {
542            vec_mut.push(tt);
543        }
544    }
545
546    /// Push `stream` onto the end of the stream, possibly gluing the first
547    /// token tree to the last token. (No other token trees will be glued.)
548    /// Uses `make_mut` to maximize efficiency.
549    pub fn push_stream(&mut self, stream: TokenStream) {
550        let vec_mut = Arc::make_mut(&mut self.0);
551
552        let stream_iter = stream.0.iter().cloned();
553
554        if let Some(first) = stream.0.first()
555            && Self::try_glue_to_last(vec_mut, first)
556        {
557            // Now skip the first token tree from `stream`.
558            vec_mut.extend(stream_iter.skip(1));
559        } else {
560            // Append all of `stream`.
561            vec_mut.extend(stream_iter);
562        }
563    }
564
565    pub fn chunks(&self, chunk_size: usize) -> core::slice::Chunks<'_, TokenTree> {
566        self.0.chunks(chunk_size)
567    }
568
569    /// Desugar doc comments like `/// foo` in the stream into `#[doc =
570    /// r"foo"]`. Modifies the `TokenStream` via `Arc::make_mut`, but as little
571    /// as possible.
572    pub fn desugar_doc_comments(&mut self) {
573        if let Some(desugared_stream) = desugar_inner(self.clone()) {
574            *self = desugared_stream;
575        }
576
577        // The return value is `None` if nothing in `stream` changed.
578        fn desugar_inner(mut stream: TokenStream) -> Option<TokenStream> {
579            let mut i = 0;
580            let mut modified = false;
581            while let Some(tt) = stream.0.get(i) {
582                match tt {
583                    &TokenTree::Token(
584                        Token { kind: token::DocComment(_, attr_style, data), span },
585                        _spacing,
586                    ) => {
587                        let desugared = desugared_tts(attr_style, data, span);
588                        let desugared_len = desugared.len();
589                        Arc::make_mut(&mut stream.0).splice(i..i + 1, desugared);
590                        modified = true;
591                        i += desugared_len;
592                    }
593
594                    &TokenTree::Token(..) => i += 1,
595
596                    &TokenTree::Delimited(sp, spacing, delim, ref delim_stream) => {
597                        if let Some(desugared_delim_stream) = desugar_inner(delim_stream.clone()) {
598                            let new_tt =
599                                TokenTree::Delimited(sp, spacing, delim, desugared_delim_stream);
600                            Arc::make_mut(&mut stream.0)[i] = new_tt;
601                            modified = true;
602                        }
603                        i += 1;
604                    }
605                }
606            }
607            if modified { Some(stream) } else { None }
608        }
609
610        fn desugared_tts(attr_style: AttrStyle, data: Symbol, span: Span) -> Vec<TokenTree> {
611            // Searches for the occurrences of `"#*` and returns the minimum number of `#`s
612            // required to wrap the text. E.g.
613            // - `abc d` is wrapped as `r"abc d"` (num_of_hashes = 0)
614            // - `abc "d"` is wrapped as `r#"abc "d""#` (num_of_hashes = 1)
615            // - `abc "##d##"` is wrapped as `r###"abc ##"d"##"###` (num_of_hashes = 3)
616            let mut num_of_hashes = 0;
617            let mut count = 0;
618            for ch in data.as_str().chars() {
619                count = match ch {
620                    '"' => 1,
621                    '#' if count > 0 => count + 1,
622                    _ => 0,
623                };
624                num_of_hashes = cmp::max(num_of_hashes, count);
625            }
626
627            // `/// foo` becomes `[doc = r"foo"]`.
628            let delim_span = DelimSpan::from_single(span);
629            let body = TokenTree::Delimited(
630                delim_span,
631                DelimSpacing::new(Spacing::JointHidden, Spacing::Alone),
632                Delimiter::Bracket,
633                [
634                    TokenTree::token_alone(token::Ident(sym::doc, token::IdentIsRaw::No), span),
635                    TokenTree::token_alone(token::Eq, span),
636                    TokenTree::token_alone(
637                        TokenKind::lit(token::StrRaw(num_of_hashes), data, None),
638                        span,
639                    ),
640                ]
641                .into_iter()
642                .collect::<TokenStream>(),
643            );
644
645            if attr_style == AttrStyle::Inner {
646                vec![
647                    TokenTree::token_joint(token::Pound, span),
648                    TokenTree::token_joint_hidden(token::Bang, span),
649                    body,
650                ]
651            } else {
652                vec![TokenTree::token_joint_hidden(token::Pound, span), body]
653            }
654        }
655    }
656}
657
658#[derive(Clone)]
659pub struct TokenStreamIter<'t> {
660    stream: &'t TokenStream,
661    index: usize,
662}
663
664impl<'t> TokenStreamIter<'t> {
665    fn new(stream: &'t TokenStream) -> Self {
666        TokenStreamIter { stream, index: 0 }
667    }
668
669    // Peeking could be done via `Peekable`, but most iterators need peeking,
670    // and this is simple and avoids the need to use `peekable` and `Peekable`
671    // at all the use sites.
672    pub fn peek(&self) -> Option<&'t TokenTree> {
673        self.stream.0.get(self.index)
674    }
675}
676
677impl<'t> Iterator for TokenStreamIter<'t> {
678    type Item = &'t TokenTree;
679
680    fn next(&mut self) -> Option<&'t TokenTree> {
681        self.stream.0.get(self.index).map(|tree| {
682            self.index += 1;
683            tree
684        })
685    }
686}
687
688#[derive(Debug, Copy, Clone, PartialEq, Encodable, Decodable, HashStable_Generic)]
689pub struct DelimSpan {
690    pub open: Span,
691    pub close: Span,
692}
693
694impl DelimSpan {
695    pub fn from_single(sp: Span) -> Self {
696        DelimSpan { open: sp, close: sp }
697    }
698
699    pub fn from_pair(open: Span, close: Span) -> Self {
700        DelimSpan { open, close }
701    }
702
703    pub fn dummy() -> Self {
704        Self::from_single(DUMMY_SP)
705    }
706
707    pub fn entire(self) -> Span {
708        self.open.with_hi(self.close.hi())
709    }
710}
711
712#[derive(Copy, Clone, Debug, PartialEq, Encodable, Decodable, HashStable_Generic)]
713pub struct DelimSpacing {
714    pub open: Spacing,
715    pub close: Spacing,
716}
717
718impl DelimSpacing {
719    pub fn new(open: Spacing, close: Spacing) -> DelimSpacing {
720        DelimSpacing { open, close }
721    }
722}
723
724// Some types are used a lot. Make sure they don't unintentionally get bigger.
725#[cfg(target_pointer_width = "64")]
726mod size_asserts {
727    use rustc_data_structures::static_assert_size;
728
729    use super::*;
730    // tidy-alphabetical-start
731    static_assert_size!(AttrTokenStream, 8);
732    static_assert_size!(AttrTokenTree, 32);
733    static_assert_size!(LazyAttrTokenStream, 8);
734    static_assert_size!(Option<LazyAttrTokenStream>, 8); // must be small, used in many AST nodes
735    static_assert_size!(TokenStream, 8);
736    static_assert_size!(TokenTree, 32);
737    // tidy-alphabetical-end
738}
rustc_ast/tokenstream.rs

rustc_ast/
tokenstream.rs
