Skip to main content

wright/lexer/
token.rs

1//! Token models.
2
3use crate::source_tracking::fragment::Fragment;
4
5/// A token in wright source code.
6#[derive(Debug, Clone)]
7pub struct Token {
8    /// What type of token this is.
9    pub variant: TokenTy,
10    /// The matching fragment of source code -- this contains the location and length data for the token.
11    pub fragment: Fragment,
12}
13
14/// The different types of tokens in wright source.
15///
16/// This enum is also used directly as the terminal type of the generated parser (see [crate::parser]), which is why
17/// every variant is field-less and the names line up with the token names in `src/parser/wright.llw`.
18#[rustfmt::skip] // Turn off auto reformat.
19#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
20// Allow missing docs (most of these should be self-evident).
21#[allow(missing_docs)]
22pub enum TokenTy {
23    LeftCurly, RightCurly,
24    LeftBracket, RightBracket,
25    LeftParen, RightParen,
26
27    Plus, PlusEq,
28    Star, StarEq,
29    Div, DivEq,
30    Xor, XorEq,
31    Mod, ModEq,
32    Bang, BangEq,
33
34    Minus, MinusEq, SingleArrow,
35    Eq, EqEq, DoubleArrow,
36
37    Lt, LtEq, LtLt, LtLtEq,
38    Gt, GtEq, GtGt, GtGtEq,
39    And, AndEq, AndAnd,
40    Or, OrEq, OrOr,
41    Colon, ColonEq, ColonColon,
42    Dot, DotDot, DotDotEq,
43
44    At,
45    Tilde,
46    Semi,
47    Comma,
48    Hash,
49    Question,
50    Dollar,
51
52    // Not in the same group as the other ones there since it can be used at the start of identifiers.
53    Underscore,
54
55    Identifier,
56
57    /// A non-documenting `// ...` comment.
58    LineComment,
59    /// A non-documenting `/* ... */` comment.
60    BlockComment,
61
62    /// A `/// ...` comment documenting the item that follows it.
63    OuterDocComment,
64    /// A `/** ... */` comment documenting the item that follows it.
65    OuterBlockDocComment,
66    /// A `//! ...` comment documenting the enclosing item (or file).
67    InnerDocComment,
68    /// A `/*! ... */` comment documenting the enclosing item (or file).
69    InnerBlockDocComment,
70
71    /// Indicates a block style comment without termination.
72    /// Separate from the other comment variants to indicate that
73    /// unterminated comments will be handled differently (they produce errors).
74    UnterminatedBlockComment,
75
76    KwRecord,
77    KwType,
78    KwEnum,
79    KwUnion,
80    KwFunc,
81    KwPure,
82    KwNaked,
83    KwUnsafe,
84    KwRepr,
85    KwImpl,
86    KwConstrain,
87    KwConstraint,
88    KwReferences,
89    KwTrait,
90    KwUse,
91    KwAs,
92    KwConst,
93    KwMod,
94    KwIf,
95    KwElse,
96    KwMatch,
97    KwFor,
98    KwIn,
99    KwWhile,
100    KwTrue,
101    KwFalse,
102    KwLoop,
103    KwWhere,
104    KwPub,
105    KwReturn,
106    KwBreak,
107    KwContinue,
108
109    KwLet,
110    KwVar,
111
112    // Keyword primitive types.
113    KwBool,
114    KwU8,
115    KwI8,
116    KwU16,
117    KwI16,
118    KwU32,
119    KwI32,
120    KwF32,
121    KwU64,
122    KwI64,
123    KwF64,
124    KwChar,
125
126    IntegerLiteral,
127    FloatLiteral,
128    StringLiteral,
129    FormatStringLiteral,
130    CharLiteral,
131
132    /// A string literal that reached the end of the source before its closing `"`.
133    UnterminatedStringLiteral,
134    /// A format string literal that reached the end of the source before its closing `` ` ``.
135    UnterminatedFormatStringLiteral,
136    /// A character literal that reached the end of the source before its closing `'`.
137    UnterminatedCharLiteral,
138
139    /// Whitespace counts as a token.
140    Whitespace,
141
142    /// Unknown character in lexer fragment.
143    Unknown,
144
145    /// End of input. Never produced by the [Lexer](crate::lexer::Lexer) -- used internally by the parser.
146    EOF,
147
148    /// End of input when parsing a single expression. Never produced by the [Lexer](crate::lexer::Lexer) -- used
149    /// internally by the parser.
150    EOFExpr,
151
152    /// End of input when parsing a single type signature. Never produced by the [Lexer](crate::lexer::Lexer) --
153    /// used internally by the parser.
154    EOFTy,
155
156    /// A token that could not be lexed. Never produced by the [Lexer](crate::lexer::Lexer) -- the parser converts
157    /// [TokenTy::Unknown] tokens into this variant, which is always skipped during parsing.
158    Error,
159}
160
161impl TokenTy {
162    /// Is this token whitespace or a comment of some kind (including doc comments and unterminated comments)?
163    pub const fn is_trivia(self) -> bool {
164        use TokenTy::*;
165
166        matches!(
167            self,
168            Whitespace
169                | LineComment
170                | BlockComment
171                | OuterDocComment
172                | OuterBlockDocComment
173                | InnerDocComment
174                | InnerBlockDocComment
175                | UnterminatedBlockComment
176        )
177    }
178
179    /// Is this token a documentation comment that documents the item following it (`///` or `/** */`)?
180    pub const fn is_outer_doc_comment(self) -> bool {
181        matches!(self, TokenTy::OuterDocComment | TokenTy::OuterBlockDocComment)
182    }
183
184    /// Is this token a documentation comment that documents the enclosing item (`//!` or `/*! */`)?
185    pub const fn is_inner_doc_comment(self) -> bool {
186        matches!(self, TokenTy::InnerDocComment | TokenTy::InnerBlockDocComment)
187    }
188}