wright/lexer/
token.rs

Help
1//! Token models.
2
3use crate::source_tracking::fragment::Fragment;
4use std::fmt::{self, Display};
5
6/// A token in wright source code.
7#[derive(Debug)]
8pub struct Token {
9    /// What type of token this is.
10    pub variant: TokenTy,
11    /// The matching fragment of source code -- this contains the location and length data for the token.
12    pub fragment: Fragment,
13}
14
15/// The different types of tokens in wright source.
16#[rustfmt::skip] // Turn off auto reformat. 
17#[derive(Clone, Copy, Debug, PartialEq, Eq)]
18// Allow missing docs (most of these should be self-evident). 
19#[allow(missing_docs)]
20pub enum TokenTy {
21    LeftCurly, RightCurly,
22    LeftBracket, RightBracket,
23    LeftParen, RightParen,
24
25    Plus, PlusEq,
26    Star, StarEq,
27    Div, DivEq,
28    Xor, XorEq,
29    Mod, ModEq,
30    Bang, BangEq,
31
32    Minus, MinusEq, SingleArrow,
33    Eq, EqEq, DoubleArrow,
34
35    Lt, LtEq, LtLt,
36    Gt, GtEq, GtGt,
37    And, AndEq, AndAnd,
38    Or, OrEq, OrOr,
39    Colon, ColonEq, ColonColon,
40
41    At,
42    Tilde,
43    Semi,
44    Dot,
45    Comma,
46    Hash,
47    Question,
48    Dollar,
49    
50    // Not in the same group as the other ones there since it can be used at the start of identifiers.
51    Underscore,
52
53    Identifier,
54
55    OuterDocComment, OuterBlockDocComment,
56    InnerDocComment, InnerBlockDocComment,
57    
58    /// Indicates a block style comment without termination. 
59    /// Separate from [TokenTy::InnerDocComment] and [TokenTy::OuterDocComment] to indicate that 
60    /// unterminated comments will be handled differently (produce errors eventually). 
61    UnterminatedBlockComment,
62
63    KwRecord,
64    KwType,
65    KwEnum,
66    KwUnion,
67    KwFunc,
68    KwPure,
69    KwNaked,
70    KwUnsafe,
71    KwRepr,
72    KwImpl,
73    KwConstrain,
74    KwConstraint,
75    KwReferences,
76    KwTrait,
77    KwUse,
78    KwAs,
79    KwConst,
80    KwMod,
81    KwIf,
82    KwElse,
83    KwMatch,
84    KwFor,
85    KwIn,
86    KwWhile,
87    KwTrue,
88    KwFalse,
89    KwLoop,
90    KwWhere,
91    KwPub,
92
93    KwLet,
94    KwVar,
95
96    // Keyword primitive types.
97    KwBool,
98    KwU8,
99    KwI8,
100    KwU16,
101    KwI16,
102    KwU32,
103    KwI32,
104    KwF32,
105    KwU64,
106    KwI64,
107    KwF64,
108    KwChar,
109
110    IntegerLiteral,
111    StringLiteral { terminated: bool }, 
112    FormatStringLiteral { terminated: bool },
113    CharLiteral { terminated: bool },
114
115    /// Whitespace counts as a token.
116    Whitespace,
117
118    /// Unknown character in lexer fragment. 
119    Unknown
120}
121
122impl Display for Token {
123    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
124        // If the host terminal supports unicode, replace the newline & carriage return characters with pictures,
125        // otherwise use ascii.
126        let replacements = match crate::util::supports_unicode::supports_unicode() {
127            true => &[("\n", "\u{240A}"), ("\r", "\u{240D}")],
128            false => &[("\n", "[nl]"), ("\r", "[cr]")],
129        };
130
131        let mut with_replacements = self.fragment.as_str().to_owned();
132
133        for (replace, replace_with) in replacements {
134            with_replacements = with_replacements.replace(replace, replace_with);
135        }
136
137        write!(f, "\"{with_replacements}\" ({:?})", self.variant)
138    }
139}
wright/lexer/token.rs

wright/lexer/
token.rs
