Skip to main content

wright/lexer/
trivial.rs

1//! Trivial tokens and their implementation.
2
3use super::{
4    Lexer,
5    token::{Token, TokenTy},
6};
7
8/// Trivial tokens that are three ASCII characters and can be matched directly
9/// against the input source code.
10pub const THREE_ASCII_TRIVIAL_TOKENS: &[(&[u8; 3], TokenTy)] = &[
11    (b"..=", TokenTy::DotDotEq),
12    (b"<<=", TokenTy::LtLtEq),
13    (b">>=", TokenTy::GtGtEq),
14];
15
16/// Trivial tokens that are two ASCII characters and can be matched directly
17/// against the input source code.
18pub const TWO_ASCII_TRIVIAL_TOKENS: &[(&[u8; 2], TokenTy)] = &[
19    (b"..", TokenTy::DotDot),
20    (b"->", TokenTy::SingleArrow),
21    (b"-=", TokenTy::MinusEq),
22    (b"=>", TokenTy::DoubleArrow),
23    (b"==", TokenTy::EqEq),
24    (b"&&", TokenTy::AndAnd),
25    (b"||", TokenTy::OrOr),
26    (b"<<", TokenTy::LtLt),
27    (b">>", TokenTy::GtGt),
28    (b"::", TokenTy::ColonColon),
29    (b"|=", TokenTy::OrEq),
30    (b"&=", TokenTy::AndEq),
31    (b":=", TokenTy::ColonEq),
32    (b">=", TokenTy::GtEq),
33    (b"<=", TokenTy::LtEq),
34    (b"!=", TokenTy::BangEq),
35    (b"%=", TokenTy::ModEq),
36    (b"^=", TokenTy::XorEq),
37    (b"*=", TokenTy::StarEq),
38    (b"+=", TokenTy::PlusEq),
39    (b"/=", TokenTy::DivEq),
40];
41
42/// Single ASCII character trivial tokens that can be matched directly against
43/// the source code.
44pub const SINGLE_ASCII_CHAR_TRIVIAL_TOKENS: &[(u8, TokenTy)] = &[
45    (b'(', TokenTy::LeftParen),
46    (b')', TokenTy::RightParen),
47    (b'[', TokenTy::LeftBracket),
48    (b']', TokenTy::RightBracket),
49    (b'{', TokenTy::LeftCurly),
50    (b'}', TokenTy::RightCurly),
51    (b'@', TokenTy::At),
52    (b';', TokenTy::Semi),
53    (b'?', TokenTy::Question),
54    (b',', TokenTy::Comma),
55    (b'#', TokenTy::Hash),
56    (b'$', TokenTy::Dollar),
57    (b'>', TokenTy::Gt),
58    (b'<', TokenTy::Lt),
59    (b'-', TokenTy::Minus),
60    (b':', TokenTy::Colon),
61    (b'!', TokenTy::Bang),
62    (b'=', TokenTy::Eq),
63    (b'&', TokenTy::And),
64    (b'|', TokenTy::Or),
65    (b'/', TokenTy::Div),
66    (b'+', TokenTy::Plus),
67    (b'^', TokenTy::Xor),
68    (b'*', TokenTy::Star),
69    (b'%', TokenTy::Mod),
70    (b'.', TokenTy::Dot),
71    (b'~', TokenTy::Tilde),
72];
73
74/// Attempt to consume a "trivial" token from the start of the [Lexer]'s [Lexer::remaining] fragment.
75///
76/// Leave the lexer unmodified if one is not available.
77pub fn try_consume_trivial_token(lexer: &mut Lexer) -> Option<Token> {
78    // Get the number of bytes remaining, since we need at least 1 to parse anything.
79    let bytes_remaining: usize = lexer.bytes_remaining();
80
81    // No token if there are no bytes of source left.
82    if bytes_remaining == 0 {
83        return None;
84    }
85
86    // Attempt to match any three-byte ASCII trivial tokens first, since matching is greedy.
87    if let Some(bytes) = lexer.remaining.as_str().as_bytes().get(0..3) {
88        for (pattern, kind) in THREE_ASCII_TRIVIAL_TOKENS {
89            if bytes == *pattern {
90                // The pattern is ASCII, so this is a valid split point.
91                return Some(lexer.split_token_unchecked(3, *kind));
92            }
93        }
94    }
95
96    // Attempt to match any two-byte ASCII trivial tokens.
97    // This must be done before single-ascii byte tokens since matching is greedy.
98    if bytes_remaining >= 2 {
99        // Get the first two bytes of the remaining fragment.
100        // SAFETY: We just checked length.
101        let bytes: &[u8] = unsafe { lexer.remaining.as_str().as_bytes().get_unchecked(0..2) };
102
103        // Match against each possible token pattern.
104        for (pattern, kind) in TWO_ASCII_TRIVIAL_TOKENS {
105            if bytes == *pattern {
106                // We have already done bounds checking, and this cannot be a character
107                // boundary since we just matched against ASCII characters.
108                return Some(lexer.split_token_unchecked(2, *kind));
109            }
110        }
111    }
112
113    // Do the same for single byte patterns.
114    // SAFETY: We checked that the number of bytes remaining is not 0 above.
115    let byte: &u8 = unsafe { lexer.remaining.as_str().as_bytes().get_unchecked(0) };
116
117    for (pattern, kind) in SINGLE_ASCII_CHAR_TRIVIAL_TOKENS {
118        if byte == pattern {
119            // If we matched, then the first byte is ASCII, and therefore we don't have to worry
120            // about bounds or unicode boundaries.
121            return Some(lexer.split_token_unchecked(1, *kind));
122        }
123    }
124
125    // If nothing else has matched, there is no trivial token available.
126    None
127}
128
129#[cfg(test)]
130mod tests {
131    use super::{Lexer, TokenTy};
132
133    #[test]
134    fn plus_and_plus_eq_tokens() {
135        let mut plus = Lexer::new_test("+");
136        let mut plus_eq = Lexer::new_test("+=");
137
138        let plus_token = plus.next_token().unwrap();
139        let plus_eq_token = plus_eq.next_token().unwrap();
140
141        assert_eq!(plus.bytes_remaining(), 0);
142        assert_eq!(plus_eq.bytes_remaining(), 0);
143        assert_eq!(plus_token.variant, TokenTy::Plus);
144        assert_eq!(plus_eq_token.variant, TokenTy::PlusEq);
145    }
146
147    #[test]
148    fn plus_one_token() {
149        let mut plus_one = Lexer::new_test("+1");
150        let plus_token = plus_one.next_token().unwrap();
151        assert_eq!(plus_one.bytes_remaining(), 1);
152        assert_eq!(plus_token.variant, TokenTy::Plus);
153        assert_eq!(plus_token.fragment.len(), 1);
154    }
155
156    #[test]
157    fn greedy_multi_char_tokens() {
158        let cases = [
159            ("..=", TokenTy::DotDotEq),
160            ("..", TokenTy::DotDot),
161            (".", TokenTy::Dot),
162            ("<<=", TokenTy::LtLtEq),
163            (">>=", TokenTy::GtGtEq),
164            (">>", TokenTy::GtGt),
165            ("~", TokenTy::Tilde),
166            ("->", TokenTy::SingleArrow),
167        ];
168
169        for (source, expected) in cases {
170            let mut lexer = Lexer::new_test(source);
171            let token = lexer.next_token().unwrap();
172            assert_eq!(token.variant, expected, "lexing {source:?}");
173            assert_eq!(lexer.bytes_remaining(), 0, "lexing {source:?}");
174        }
175    }
176}