Struct Lexer

struct Lexer<'psess, 'src> {
    psess: &'psess ParseSess,
    start_pos: BytePos,
    pos: BytePos,
    src: &'src str,
    cursor: Cursor<'src>,
    override_span: Option<Span>,
    nbsp_is_whitespace: bool,
    last_lifetime: Option<Span>,
    token: Token,
    diag_info: TokenTreeDiagInfo,
}

Fields

psess: &'psess ParseSess

start_pos: BytePos

Initial position, read-only.

pos: BytePos

The absolute offset within the source_map of the current character.

src: &'src str

Source text to tokenize.

cursor: Cursor<'src>

Cursor for getting lexer tokens.

override_span: Option<Span>

nbsp_is_whitespace: bool

When a “unknown start of token: \u{a0}” has already been emitted earlier in this file, it’s safe to treat further occurrences of the non-breaking space character as whitespace.

last_lifetime: Option<Span>

Track the Span for the leading ' of the last lifetime. Used for diagnostics to detect possible typo where " was meant.

token: Token

The current token.

diag_info: TokenTreeDiagInfo

Implementations

impl<'psess, 'src> Lexer<'psess, 'src>

pub(super) fn lex_token_trees( &mut self, is_delimited: bool, ) -> Result<(Spacing, TokenStream), Vec<Diag<'psess>>>

fn eof_err(&mut self) -> Diag<'psess>

fn lex_token_tree_open_delim( &mut self, open_delim: Delimiter, ) -> Result<TokenTree, Vec<Diag<'psess>>>

fn bump(&mut self) -> (Token, Spacing)

fn bump_minimal(&mut self) -> Spacing

fn close_delim_err(&mut self, delim: Delimiter) -> Diag<'psess>

impl<'psess, 'src> Lexer<'psess, 'src>

fn dcx(&self) -> DiagCtxtHandle<'psess>

fn mk_sp(&self, lo: BytePos, hi: BytePos) -> Span

fn next_token_from_cursor(&mut self) -> (Token, bool)

Returns the next token, paired with a bool indicating if the token was preceded by whitespace.

fn ident(&self, start: BytePos) -> TokenKind

fn lint_unicode_text_flow(&self, start: BytePos)

Detect usages of Unicode codepoints changing the direction of the text on screen and loudly complain about it.

fn cook_doc_comment( &self, content_start: BytePos, content: &str, comment_kind: CommentKind, doc_style: DocStyle, ) -> TokenKind

fn cook_lexer_literal( &self, start: BytePos, end: BytePos, kind: LiteralKind, ) -> (LitKind, Symbol)

fn src_index(&self, pos: BytePos) -> usize

fn str_from(&self, start: BytePos) -> &'src str

Slice of the source text from start up to but excluding self.pos, meaning the slice does not include the character self.ch.

fn symbol_from_to(&self, start: BytePos, end: BytePos) -> Symbol

As symbol_from, with an explicit endpoint.

fn str_from_to(&self, start: BytePos, end: BytePos) -> &'src str

Slice of the source text spanning from start up to but excluding end.

fn str_from_to_end(&self, start: BytePos) -> &'src str

Slice of the source text spanning from start until the end

fn report_raw_str_error(&self, start: BytePos, prefix_len: u32) -> !

fn report_non_started_raw_string(&self, start: BytePos, bad_char: char) -> !

fn report_unterminated_raw_string( &self, start: BytePos, n_hashes: u32, possible_offset: Option<u32>, found_terminators: u32, ) -> !

fn report_unterminated_block_comment( &self, start: BytePos, doc_style: Option<DocStyle>, )

fn report_unknown_prefix(&self, start: BytePos)

fn maybe_report_guarded_str( &mut self, start: BytePos, str_before: &'src str, ) -> TokenKind

Detect guarded string literal syntax

RFC 3593 reserved this syntax for future use. As of Rust 2024, using this syntax produces an error. In earlier editions, however, it only results in an (allowed by default) lint, and is treated as separate tokens.

fn report_too_many_hashes(&self, start: BytePos, num: u32) -> !

fn cook_common( &self, kind: LitKind, mode: Mode, start: BytePos, end: BytePos, prefix_len: u32, postfix_len: u32, unescape: fn(_: &str, _: Mode, _: &mut dyn FnMut(Range<usize>, Result<(), EscapeError>)), ) -> (LitKind, Symbol)

fn cook_unicode( &self, kind: LitKind, mode: Mode, start: BytePos, end: BytePos, prefix_len: u32, postfix_len: u32, ) -> (LitKind, Symbol)

fn cook_mixed( &self, kind: LitKind, mode: Mode, start: BytePos, end: BytePos, prefix_len: u32, postfix_len: u32, ) -> (LitKind, Symbol)

Layout

Note: Most layout information is completely unstable and may even differ between compilations. The only exception is types with certain repr(...) attributes. Please see the Rust Reference's “Type Layout” chapter for details on type layout guarantees.

Size: 224 bytes