wright/lexer/token.rs
1//! Token models.
2
3use crate::source_tracking::fragment::Fragment;
4
5/// A token in wright source code.
6#[derive(Debug, Clone)]
7pub struct Token {
8 /// What type of token this is.
9 pub variant: TokenTy,
10 /// The matching fragment of source code -- this contains the location and length data for the token.
11 pub fragment: Fragment,
12}
13
14/// The different types of tokens in wright source.
15///
16/// This enum is also used directly as the terminal type of the generated parser (see [crate::parser]), which is why
17/// every variant is field-less and the names line up with the token names in `src/parser/wright.llw`.
18#[rustfmt::skip] // Turn off auto reformat.
19#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
20// Allow missing docs (most of these should be self-evident).
21#[allow(missing_docs)]
22pub enum TokenTy {
23 LeftCurly, RightCurly,
24 LeftBracket, RightBracket,
25 LeftParen, RightParen,
26
27 Plus, PlusEq,
28 Star, StarEq,
29 Div, DivEq,
30 Xor, XorEq,
31 Mod, ModEq,
32 Bang, BangEq,
33
34 Minus, MinusEq, SingleArrow,
35 Eq, EqEq, DoubleArrow,
36
37 Lt, LtEq, LtLt, LtLtEq,
38 Gt, GtEq, GtGt, GtGtEq,
39 And, AndEq, AndAnd,
40 Or, OrEq, OrOr,
41 Colon, ColonEq, ColonColon,
42 Dot, DotDot, DotDotEq,
43
44 At,
45 Tilde,
46 Semi,
47 Comma,
48 Hash,
49 Question,
50 Dollar,
51
52 // Not in the same group as the other ones there since it can be used at the start of identifiers.
53 Underscore,
54
55 Identifier,
56
57 /// A non-documenting `// ...` comment.
58 LineComment,
59 /// A non-documenting `/* ... */` comment.
60 BlockComment,
61
62 /// A `/// ...` comment documenting the item that follows it.
63 OuterDocComment,
64 /// A `/** ... */` comment documenting the item that follows it.
65 OuterBlockDocComment,
66 /// A `//! ...` comment documenting the enclosing item (or file).
67 InnerDocComment,
68 /// A `/*! ... */` comment documenting the enclosing item (or file).
69 InnerBlockDocComment,
70
71 /// Indicates a block style comment without termination.
72 /// Separate from the other comment variants to indicate that
73 /// unterminated comments will be handled differently (they produce errors).
74 UnterminatedBlockComment,
75
76 KwRecord,
77 KwType,
78 KwEnum,
79 KwUnion,
80 KwFunc,
81 KwPure,
82 KwNaked,
83 KwUnsafe,
84 KwRepr,
85 KwImpl,
86 KwConstrain,
87 KwConstraint,
88 KwReferences,
89 KwTrait,
90 KwUse,
91 KwAs,
92 KwConst,
93 KwMod,
94 KwIf,
95 KwElse,
96 KwMatch,
97 KwFor,
98 KwIn,
99 KwWhile,
100 KwTrue,
101 KwFalse,
102 KwLoop,
103 KwWhere,
104 KwPub,
105 KwReturn,
106 KwBreak,
107 KwContinue,
108
109 KwLet,
110 KwVar,
111
112 // Keyword primitive types.
113 KwBool,
114 KwU8,
115 KwI8,
116 KwU16,
117 KwI16,
118 KwU32,
119 KwI32,
120 KwF32,
121 KwU64,
122 KwI64,
123 KwF64,
124 KwChar,
125
126 IntegerLiteral,
127 FloatLiteral,
128 StringLiteral,
129 FormatStringLiteral,
130 CharLiteral,
131
132 /// A string literal that reached the end of the source before its closing `"`.
133 UnterminatedStringLiteral,
134 /// A format string literal that reached the end of the source before its closing `` ` ``.
135 UnterminatedFormatStringLiteral,
136 /// A character literal that reached the end of the source before its closing `'`.
137 UnterminatedCharLiteral,
138
139 /// Whitespace counts as a token.
140 Whitespace,
141
142 /// Unknown character in lexer fragment.
143 Unknown,
144
145 /// End of input. Never produced by the [Lexer](crate::lexer::Lexer) -- used internally by the parser.
146 EOF,
147
148 /// End of input when parsing a single expression. Never produced by the [Lexer](crate::lexer::Lexer) -- used
149 /// internally by the parser.
150 EOFExpr,
151
152 /// End of input when parsing a single type signature. Never produced by the [Lexer](crate::lexer::Lexer) --
153 /// used internally by the parser.
154 EOFTy,
155
156 /// A token that could not be lexed. Never produced by the [Lexer](crate::lexer::Lexer) -- the parser converts
157 /// [TokenTy::Unknown] tokens into this variant, which is always skipped during parsing.
158 Error,
159}
160
161impl TokenTy {
162 /// Is this token whitespace or a comment of some kind (including doc comments and unterminated comments)?
163 pub const fn is_trivia(self) -> bool {
164 use TokenTy::*;
165
166 matches!(
167 self,
168 Whitespace
169 | LineComment
170 | BlockComment
171 | OuterDocComment
172 | OuterBlockDocComment
173 | InnerDocComment
174 | InnerBlockDocComment
175 | UnterminatedBlockComment
176 )
177 }
178
179 /// Is this token a documentation comment that documents the item following it (`///` or `/** */`)?
180 pub const fn is_outer_doc_comment(self) -> bool {
181 matches!(self, TokenTy::OuterDocComment | TokenTy::OuterBlockDocComment)
182 }
183
184 /// Is this token a documentation comment that documents the enclosing item (`//!` or `/*! */`)?
185 pub const fn is_inner_doc_comment(self) -> bool {
186 matches!(self, TokenTy::InnerDocComment | TokenTy::InnerBlockDocComment)
187 }
188}