Enum rustc_lexer::TokenKind

source · [−]

pub enum TokenKind {
Show 37 variants    LineComment {
        doc_style: Option<DocStyle>,
    },
    BlockComment {
        doc_style: Option<DocStyle>,
        terminated: bool,
    },
    Whitespace,
    Ident,
    InvalidIdent,
    RawIdent,
    UnknownPrefix,
    Literal {
        kind: LiteralKind,
        suffix_start: usize,
    },
    Lifetime {
        starts_with_number: bool,
    },
    Semi,
    Comma,
    Dot,
    OpenParen,
    CloseParen,
    OpenBrace,
    CloseBrace,
    OpenBracket,
    CloseBracket,
    At,
    Pound,
    Tilde,
    Question,
    Colon,
    Dollar,
    Eq,
    Bang,
    Lt,
    Gt,
    Minus,
    And,
    Or,
    Plus,
    Star,
    Slash,
    Caret,
    Percent,
    Unknown,
}

Expand description

Enum representing common lexeme types.

Variants

`LineComment`

Fields

doc_style: Option<DocStyle>

“// comment”

`BlockComment`

Fields

doc_style: Option<DocStyle>

terminated: bool

/* block comment */

Block comments can be recursive, so the sequence like /* /* */ will not be considered terminated and will result in a parsing error.

`Whitespace`

Any whitespace characters sequence.

`Ident`

“ident” or “continue” At this step keywords are also considered identifiers.

`InvalidIdent`

Like the above, but containing invalid unicode codepoints.

`RawIdent`

“r#ident”

`UnknownPrefix`

An unknown prefix like foo#, foo', foo". Note that only the prefix (foo) is included in the token, not the separator (which is lexed as its own distinct token). In Rust 2021 and later, reserved prefixes are reported as errors; in earlier editions, they result in a (allowed by default) lint, and are treated as regular identifier tokens.

`Literal`

Fields

kind: LiteralKind

suffix_start: usize

“12_u8”, “1.0e-40”, “b“123”“. See LiteralKind for more details.

`Lifetime`

Fields

starts_with_number: bool

“’a”

`Semi`

“;”

`Comma`

“,”

`Dot`

“.”

`OpenParen`

“(”

`CloseParen`

“)”

`OpenBrace`

“{”

`CloseBrace`

“}”

`OpenBracket`

“[”

`CloseBracket`

“]”

`At`

“@”

`Pound`

“#”

`Tilde`

“~”

`Question`

“?”

`Colon`

“:”

`Dollar`

“$”

`Eq`

“=”

`Bang`

“!”

`Lt`

“<”

`Gt`

“>”

`Minus`

“-”

`And`

“&”

`Or`

“|”

`Plus`

“+”

`Star`

“*”

`Slash`

“/”

`Caret`

“^”

`Percent`

“%”

`Unknown`

Unknown token, not expected by the lexer, e.g. “№”

Enum rustc_lexer::TokenKind

Variants

LineComment

Fields

BlockComment

Fields

Whitespace

Ident

InvalidIdent

RawIdent

UnknownPrefix

Literal

Fields

Lifetime

Fields

Semi

Comma

Dot

OpenParen

CloseParen

OpenBrace

CloseBrace

OpenBracket

CloseBracket

At

Pound

Tilde

Question

Colon

Dollar

Eq

Bang

Lt

Gt

Minus

And

Or

Plus

Star

Slash

Caret

Percent

Unknown

Trait Implementations

impl Clone for TokenKind

fn clone(&self) -> TokenKind

fn clone_from(&mut self, source: &Self)

impl Debug for TokenKind

fn fmt(&self, f: &mut Formatter<'_>) -> Result

impl Ord for TokenKind

fn cmp(&self, other: &TokenKind) -> Ordering

fn max(self, other: Self) -> Self

fn min(self, other: Self) -> Self

fn clamp(self, min: Self, max: Self) -> Self

impl PartialEq<TokenKind> for TokenKind

fn eq(&self, other: &TokenKind) -> bool

fn ne(&self, other: &TokenKind) -> bool

impl PartialOrd<TokenKind> for TokenKind

fn partial_cmp(&self, other: &TokenKind) -> Option<Ordering>

fn lt(&self, other: &Rhs) -> bool

fn le(&self, other: &Rhs) -> bool

fn gt(&self, other: &Rhs) -> bool

fn ge(&self, other: &Rhs) -> bool

impl Copy for TokenKind

impl Eq for TokenKind

impl StructuralEq for TokenKind

impl StructuralPartialEq for TokenKind

Auto Trait Implementations

impl RefUnwindSafe for TokenKind

impl Send for TokenKind

impl Sync for TokenKind

impl Unpin for TokenKind

impl UnwindSafe for TokenKind

Blanket Implementations

impl<T> Any for T where T: 'static + ?Sized,

fn type_id(&self) -> TypeId

impl<T> Borrow<T> for T where T: ?Sized,

fn borrow(&self) -> &T

impl<T> BorrowMut<T> for T where T: ?Sized,

fn borrow_mut(&mut self) -> &mut T

`LineComment`

`BlockComment`

`Whitespace`

`Ident`

`InvalidIdent`

`RawIdent`

`UnknownPrefix`

`Literal`

`Lifetime`

`Semi`

`Comma`

`Dot`

`OpenParen`

`CloseParen`

`OpenBrace`

`CloseBrace`

`OpenBracket`

`CloseBracket`

`At`

`Pound`

`Tilde`

`Question`

`Colon`

`Dollar`

`Eq`

`Bang`

`Lt`

`Gt`

`Minus`

`And`

`Or`

`Plus`

`Star`

`Slash`

`Caret`

`Percent`

`Unknown`

impl<T> Any for T where
T: 'static + ?Sized,

impl<T> Borrow<T> for T where
T: ?Sized,

impl<T> BorrowMut<T> for T where
T: ?Sized,

impl<T, U> Into<U> for T where
U: From<T>,

impl<T> ToOwned for T where
T: Clone,

impl<T, U> TryFrom<U> for T where
U: Into<T>,

impl<T, U> TryInto<U> for T where
U: TryFrom<T>,