rustc_ast/util/
literal.rs

Help
1//! Code related to parsing literals.
2
3use std::{ascii, fmt, str};
4
5use rustc_lexer::unescape::{
6    MixedUnit, Mode, byte_from_char, unescape_byte, unescape_char, unescape_mixed, unescape_unicode,
7};
8use rustc_span::{Span, Symbol, kw, sym};
9use tracing::debug;
10
11use crate::ast::{self, LitKind, MetaItemLit, StrStyle};
12use crate::token::{self, Token};
13
14// Escapes a string, represented as a symbol. Reuses the original symbol,
15// avoiding interning, if no changes are required.
16pub fn escape_string_symbol(symbol: Symbol) -> Symbol {
17    let s = symbol.as_str();
18    let escaped = s.escape_default().to_string();
19    if s == escaped { symbol } else { Symbol::intern(&escaped) }
20}
21
22// Escapes a char.
23pub fn escape_char_symbol(ch: char) -> Symbol {
24    let s: String = ch.escape_default().map(Into::<char>::into).collect();
25    Symbol::intern(&s)
26}
27
28// Escapes a byte string.
29pub fn escape_byte_str_symbol(bytes: &[u8]) -> Symbol {
30    let s = bytes.escape_ascii().to_string();
31    Symbol::intern(&s)
32}
33
34#[derive(Debug)]
35pub enum LitError {
36    InvalidSuffix(Symbol),
37    InvalidIntSuffix(Symbol),
38    InvalidFloatSuffix(Symbol),
39    NonDecimalFloat(u32), // u32 is the base
40    IntTooLarge(u32),     // u32 is the base
41}
42
43impl LitKind {
44    /// Converts literal token into a semantic literal.
45    pub fn from_token_lit(lit: token::Lit) -> Result<LitKind, LitError> {
46        let token::Lit { kind, symbol, suffix } = lit;
47        if let Some(suffix) = suffix
48            && !kind.may_have_suffix()
49        {
50            return Err(LitError::InvalidSuffix(suffix));
51        }
52
53        // For byte/char/string literals, chars and escapes have already been
54        // checked in the lexer (in `cook_lexer_literal`). So we can assume all
55        // chars and escapes are valid here.
56        Ok(match kind {
57            token::Bool => {
58                assert!(symbol.is_bool_lit());
59                LitKind::Bool(symbol == kw::True)
60            }
61            token::Byte => {
62                return unescape_byte(symbol.as_str())
63                    .map(LitKind::Byte)
64                    .map_err(|_| panic!("failed to unescape byte literal"));
65            }
66            token::Char => {
67                return unescape_char(symbol.as_str())
68                    .map(LitKind::Char)
69                    .map_err(|_| panic!("failed to unescape char literal"));
70            }
71
72            // There are some valid suffixes for integer and float literals,
73            // so all the handling is done internally.
74            token::Integer => return integer_lit(symbol, suffix),
75            token::Float => return float_lit(symbol, suffix),
76
77            token::Str => {
78                // If there are no characters requiring special treatment we can
79                // reuse the symbol from the token. Otherwise, we must generate a
80                // new symbol because the string in the LitKind is different to the
81                // string in the token.
82                let s = symbol.as_str();
83                // Vanilla strings are so common we optimize for the common case where no chars
84                // requiring special behaviour are present.
85                let symbol = if s.contains('\\') {
86                    let mut buf = String::with_capacity(s.len());
87                    // Force-inlining here is aggressive but the closure is
88                    // called on every char in the string, so it can be hot in
89                    // programs with many long strings containing escapes.
90                    unescape_unicode(
91                        s,
92                        Mode::Str,
93                        &mut #[inline(always)]
94                        |_, c| match c {
95                            Ok(c) => buf.push(c),
96                            Err(err) => {
97                                assert!(!err.is_fatal(), "failed to unescape string literal")
98                            }
99                        },
100                    );
101                    Symbol::intern(&buf)
102                } else {
103                    symbol
104                };
105                LitKind::Str(symbol, ast::StrStyle::Cooked)
106            }
107            token::StrRaw(n) => {
108                // Raw strings have no escapes so no work is needed here.
109                LitKind::Str(symbol, ast::StrStyle::Raw(n))
110            }
111            token::ByteStr => {
112                let s = symbol.as_str();
113                let mut buf = Vec::with_capacity(s.len());
114                unescape_unicode(s, Mode::ByteStr, &mut |_, c| match c {
115                    Ok(c) => buf.push(byte_from_char(c)),
116                    Err(err) => {
117                        assert!(!err.is_fatal(), "failed to unescape string literal")
118                    }
119                });
120                LitKind::ByteStr(buf.into(), StrStyle::Cooked)
121            }
122            token::ByteStrRaw(n) => {
123                // Raw strings have no escapes so we can convert the symbol
124                // directly to a `Arc<u8>`.
125                let buf = symbol.as_str().to_owned().into_bytes();
126                LitKind::ByteStr(buf.into(), StrStyle::Raw(n))
127            }
128            token::CStr => {
129                let s = symbol.as_str();
130                let mut buf = Vec::with_capacity(s.len());
131                unescape_mixed(s, Mode::CStr, &mut |_span, c| match c {
132                    Ok(MixedUnit::Char(c)) => {
133                        buf.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes())
134                    }
135                    Ok(MixedUnit::HighByte(b)) => buf.push(b),
136                    Err(err) => {
137                        assert!(!err.is_fatal(), "failed to unescape C string literal")
138                    }
139                });
140                buf.push(0);
141                LitKind::CStr(buf.into(), StrStyle::Cooked)
142            }
143            token::CStrRaw(n) => {
144                // Raw strings have no escapes so we can convert the symbol
145                // directly to a `Arc<u8>` after appending the terminating NUL
146                // char.
147                let mut buf = symbol.as_str().to_owned().into_bytes();
148                buf.push(0);
149                LitKind::CStr(buf.into(), StrStyle::Raw(n))
150            }
151            token::Err(guar) => LitKind::Err(guar),
152        })
153    }
154}
155
156impl fmt::Display for LitKind {
157    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
158        match *self {
159            LitKind::Byte(b) => {
160                let b: String = ascii::escape_default(b).map(Into::<char>::into).collect();
161                write!(f, "b'{b}'")?;
162            }
163            LitKind::Char(ch) => write!(f, "'{}'", escape_char_symbol(ch))?,
164            LitKind::Str(sym, StrStyle::Cooked) => write!(f, "\"{}\"", escape_string_symbol(sym))?,
165            LitKind::Str(sym, StrStyle::Raw(n)) => write!(
166                f,
167                "r{delim}\"{string}\"{delim}",
168                delim = "#".repeat(n as usize),
169                string = sym
170            )?,
171            LitKind::ByteStr(ref bytes, StrStyle::Cooked) => {
172                write!(f, "b\"{}\"", escape_byte_str_symbol(bytes))?
173            }
174            LitKind::ByteStr(ref bytes, StrStyle::Raw(n)) => {
175                // Unwrap because raw byte string literals can only contain ASCII.
176                let symbol = str::from_utf8(bytes).unwrap();
177                write!(
178                    f,
179                    "br{delim}\"{string}\"{delim}",
180                    delim = "#".repeat(n as usize),
181                    string = symbol
182                )?;
183            }
184            LitKind::CStr(ref bytes, StrStyle::Cooked) => {
185                write!(f, "c\"{}\"", escape_byte_str_symbol(bytes))?
186            }
187            LitKind::CStr(ref bytes, StrStyle::Raw(n)) => {
188                // This can only be valid UTF-8.
189                let symbol = str::from_utf8(bytes).unwrap();
190                write!(f, "cr{delim}\"{symbol}\"{delim}", delim = "#".repeat(n as usize),)?;
191            }
192            LitKind::Int(n, ty) => {
193                write!(f, "{n}")?;
194                match ty {
195                    ast::LitIntType::Unsigned(ty) => write!(f, "{}", ty.name())?,
196                    ast::LitIntType::Signed(ty) => write!(f, "{}", ty.name())?,
197                    ast::LitIntType::Unsuffixed => {}
198                }
199            }
200            LitKind::Float(symbol, ty) => {
201                write!(f, "{symbol}")?;
202                match ty {
203                    ast::LitFloatType::Suffixed(ty) => write!(f, "{}", ty.name())?,
204                    ast::LitFloatType::Unsuffixed => {}
205                }
206            }
207            LitKind::Bool(b) => write!(f, "{}", if b { "true" } else { "false" })?,
208            LitKind::Err(_) => {
209                // This only shows up in places like `-Zunpretty=hir` output, so we
210                // don't bother to produce something useful.
211                write!(f, "<bad-literal>")?;
212            }
213        }
214
215        Ok(())
216    }
217}
218
219impl MetaItemLit {
220    /// Converts a token literal into a meta item literal.
221    pub fn from_token_lit(token_lit: token::Lit, span: Span) -> Result<MetaItemLit, LitError> {
222        Ok(MetaItemLit {
223            symbol: token_lit.symbol,
224            suffix: token_lit.suffix,
225            kind: LitKind::from_token_lit(token_lit)?,
226            span,
227        })
228    }
229
230    /// Cheaply converts a meta item literal into a token literal.
231    pub fn as_token_lit(&self) -> token::Lit {
232        let kind = match self.kind {
233            LitKind::Bool(_) => token::Bool,
234            LitKind::Str(_, ast::StrStyle::Cooked) => token::Str,
235            LitKind::Str(_, ast::StrStyle::Raw(n)) => token::StrRaw(n),
236            LitKind::ByteStr(_, ast::StrStyle::Cooked) => token::ByteStr,
237            LitKind::ByteStr(_, ast::StrStyle::Raw(n)) => token::ByteStrRaw(n),
238            LitKind::CStr(_, ast::StrStyle::Cooked) => token::CStr,
239            LitKind::CStr(_, ast::StrStyle::Raw(n)) => token::CStrRaw(n),
240            LitKind::Byte(_) => token::Byte,
241            LitKind::Char(_) => token::Char,
242            LitKind::Int(..) => token::Integer,
243            LitKind::Float(..) => token::Float,
244            LitKind::Err(guar) => token::Err(guar),
245        };
246
247        token::Lit::new(kind, self.symbol, self.suffix)
248    }
249
250    /// Converts an arbitrary token into meta item literal.
251    pub fn from_token(token: &Token) -> Option<MetaItemLit> {
252        token::Lit::from_token(token)
253            .and_then(|token_lit| MetaItemLit::from_token_lit(token_lit, token.span).ok())
254    }
255}
256
257fn strip_underscores(symbol: Symbol) -> Symbol {
258    // Do not allocate a new string unless necessary.
259    let s = symbol.as_str();
260    if s.contains('_') {
261        let mut s = s.to_string();
262        s.retain(|c| c != '_');
263        return Symbol::intern(&s);
264    }
265    symbol
266}
267
268fn filtered_float_lit(
269    symbol: Symbol,
270    suffix: Option<Symbol>,
271    base: u32,
272) -> Result<LitKind, LitError> {
273    debug!("filtered_float_lit: {:?}, {:?}, {:?}", symbol, suffix, base);
274    if base != 10 {
275        return Err(LitError::NonDecimalFloat(base));
276    }
277    Ok(match suffix {
278        Some(suffix) => LitKind::Float(
279            symbol,
280            ast::LitFloatType::Suffixed(match suffix {
281                sym::f16 => ast::FloatTy::F16,
282                sym::f32 => ast::FloatTy::F32,
283                sym::f64 => ast::FloatTy::F64,
284                sym::f128 => ast::FloatTy::F128,
285                _ => return Err(LitError::InvalidFloatSuffix(suffix)),
286            }),
287        ),
288        None => LitKind::Float(symbol, ast::LitFloatType::Unsuffixed),
289    })
290}
291
292fn float_lit(symbol: Symbol, suffix: Option<Symbol>) -> Result<LitKind, LitError> {
293    debug!("float_lit: {:?}, {:?}", symbol, suffix);
294    filtered_float_lit(strip_underscores(symbol), suffix, 10)
295}
296
297fn integer_lit(symbol: Symbol, suffix: Option<Symbol>) -> Result<LitKind, LitError> {
298    debug!("integer_lit: {:?}, {:?}", symbol, suffix);
299    let symbol = strip_underscores(symbol);
300    let s = symbol.as_str();
301
302    let base = match s.as_bytes() {
303        [b'0', b'x', ..] => 16,
304        [b'0', b'o', ..] => 8,
305        [b'0', b'b', ..] => 2,
306        _ => 10,
307    };
308
309    let ty = match suffix {
310        Some(suf) => match suf {
311            sym::isize => ast::LitIntType::Signed(ast::IntTy::Isize),
312            sym::i8 => ast::LitIntType::Signed(ast::IntTy::I8),
313            sym::i16 => ast::LitIntType::Signed(ast::IntTy::I16),
314            sym::i32 => ast::LitIntType::Signed(ast::IntTy::I32),
315            sym::i64 => ast::LitIntType::Signed(ast::IntTy::I64),
316            sym::i128 => ast::LitIntType::Signed(ast::IntTy::I128),
317            sym::usize => ast::LitIntType::Unsigned(ast::UintTy::Usize),
318            sym::u8 => ast::LitIntType::Unsigned(ast::UintTy::U8),
319            sym::u16 => ast::LitIntType::Unsigned(ast::UintTy::U16),
320            sym::u32 => ast::LitIntType::Unsigned(ast::UintTy::U32),
321            sym::u64 => ast::LitIntType::Unsigned(ast::UintTy::U64),
322            sym::u128 => ast::LitIntType::Unsigned(ast::UintTy::U128),
323            // `1f64` and `2f32` etc. are valid float literals, and
324            // `fxxx` looks more like an invalid float literal than invalid integer literal.
325            _ if suf.as_str().starts_with('f') => return filtered_float_lit(symbol, suffix, base),
326            _ => return Err(LitError::InvalidIntSuffix(suf)),
327        },
328        _ => ast::LitIntType::Unsuffixed,
329    };
330
331    let s = &s[if base != 10 { 2 } else { 0 }..];
332    u128::from_str_radix(s, base)
333        .map(|i| LitKind::Int(i.into(), ty))
334        .map_err(|_| LitError::IntTooLarge(base))
335}
rustc_ast/util/literal.rs

rustc_ast/util/
literal.rs
