wright/lexer/
token.rs

1//! Token models.
2
3use crate::source_tracking::fragment::Fragment;
4use std::fmt::{self, Display};
5
6/// A token in wright source code.
7#[derive(Debug)]
8pub struct Token {
9    /// What type of token this is.
10    pub variant: TokenTy,
11    /// The matching fragment of source code -- this contains the location and length data for the token.
12    pub fragment: Fragment,
13}
14
15/// The different types of tokens in wright source.
16#[rustfmt::skip] // Turn off auto reformat. 
17#[derive(Clone, Copy, Debug, PartialEq, Eq)]
18// Allow missing docs (most of these should be self-evident). 
19#[allow(missing_docs)]
20pub enum TokenTy {
21    LeftCurly, RightCurly,
22    LeftBracket, RightBracket,
23    LeftParen, RightParen,
24
25    Plus, PlusEq,
26    Star, StarEq,
27    Div, DivEq,
28    Xor, XorEq,
29    Mod, ModEq,
30    Bang, BangEq,
31
32    Minus, MinusEq, SingleArrow,
33    Eq, EqEq, DoubleArrow,
34
35    Lt, LtEq, LtLt,
36    Gt, GtEq, GtGt,
37    And, AndEq, AndAnd,
38    Or, OrEq, OrOr,
39    Colon, ColonEq, ColonColon,
40
41    At,
42    Tilde,
43    Semi,
44    Dot,
45    Comma,
46    Hash,
47    Question,
48    Dollar,
49    
50    // Not in the same group as the other ones there since it can be used at the start of identifiers.
51    Underscore,
52
53    Identifier,
54
55    OuterDocComment, OuterBlockDocComment,
56    InnerDocComment, InnerBlockDocComment,
57    
58    /// Indicates a block style comment without termination. 
59    /// Separate from [TokenTy::InnerDocComment] and [TokenTy::OuterDocComment] to indicate that 
60    /// unterminated comments will be handled differently (produce errors eventually). 
61    UnterminatedBlockComment,
62
63    KwRecord,
64    KwType,
65    KwEnum,
66    KwUnion,
67    KwFunc,
68    KwPure,
69    KwRepr,
70    KwImpl,
71    KwConstraint,
72    KwReferences,
73    KwTrait,
74    KwUse,
75    KwAs,
76    KwConst,
77    KwMod,
78    KwIf,
79    KwElse,
80    KwMatch,
81    KwFor,
82    KwIn,
83    KwWhile,
84    KwTrue,
85    KwFalse,
86    KwLoop,
87    KwWhere,
88    KwPub,
89
90    KwLet,
91    KwVar,
92
93    // Keyword primitive types.
94    KwBool,
95    KwU8,
96    KwI8,
97    KwU16,
98    KwI16,
99    KwU32,
100    KwI32,
101    KwF32,
102    KwU64,
103    KwI64,
104    KwF64,
105    KwChar,
106
107    IntegerLiteral,
108    StringLiteral { terminated: bool }, 
109    FormatStringLiteral { terminated: bool },
110    CharLiteral { terminated: bool },
111
112    /// Whitespace counts as a token.
113    Whitespace,
114
115    /// Unknown character in lexer fragment. 
116    Unknown
117}
118
119impl Display for Token {
120    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
121        // If the host terminal supports unicode, replace the newline & carriage return characters with pictures,
122        // otherwise use ascii.
123        let replacements = match crate::util::supports_unicode::supports_unicode() {
124            true => &[("\n", "\u{240A}"), ("\r", "\u{240D}")],
125            false => &[("\n", "[nl]"), ("\r", "[cr]")],
126        };
127
128        let mut with_replacements = self.fragment.as_str().to_owned();
129
130        for (replace, replace_with) in replacements {
131            with_replacements = with_replacements.replace(replace, replace_with);
132        }
133
134        write!(f, "\"{with_replacements}\" ({:?})", self.variant)
135    }
136}
wright/lexer/token.rs

wright/lexer/
token.rs
