1use super::{
4 Lexer,
5 token::{Token, TokenTy},
6};
7
8pub const THREE_ASCII_TRIVIAL_TOKENS: &[(&[u8; 3], TokenTy)] = &[
11 (b"..=", TokenTy::DotDotEq),
12 (b"<<=", TokenTy::LtLtEq),
13 (b">>=", TokenTy::GtGtEq),
14];
15
16pub const TWO_ASCII_TRIVIAL_TOKENS: &[(&[u8; 2], TokenTy)] = &[
19 (b"..", TokenTy::DotDot),
20 (b"->", TokenTy::SingleArrow),
21 (b"-=", TokenTy::MinusEq),
22 (b"=>", TokenTy::DoubleArrow),
23 (b"==", TokenTy::EqEq),
24 (b"&&", TokenTy::AndAnd),
25 (b"||", TokenTy::OrOr),
26 (b"<<", TokenTy::LtLt),
27 (b">>", TokenTy::GtGt),
28 (b"::", TokenTy::ColonColon),
29 (b"|=", TokenTy::OrEq),
30 (b"&=", TokenTy::AndEq),
31 (b":=", TokenTy::ColonEq),
32 (b">=", TokenTy::GtEq),
33 (b"<=", TokenTy::LtEq),
34 (b"!=", TokenTy::BangEq),
35 (b"%=", TokenTy::ModEq),
36 (b"^=", TokenTy::XorEq),
37 (b"*=", TokenTy::StarEq),
38 (b"+=", TokenTy::PlusEq),
39 (b"/=", TokenTy::DivEq),
40];
41
42pub const SINGLE_ASCII_CHAR_TRIVIAL_TOKENS: &[(u8, TokenTy)] = &[
45 (b'(', TokenTy::LeftParen),
46 (b')', TokenTy::RightParen),
47 (b'[', TokenTy::LeftBracket),
48 (b']', TokenTy::RightBracket),
49 (b'{', TokenTy::LeftCurly),
50 (b'}', TokenTy::RightCurly),
51 (b'@', TokenTy::At),
52 (b';', TokenTy::Semi),
53 (b'?', TokenTy::Question),
54 (b',', TokenTy::Comma),
55 (b'#', TokenTy::Hash),
56 (b'$', TokenTy::Dollar),
57 (b'>', TokenTy::Gt),
58 (b'<', TokenTy::Lt),
59 (b'-', TokenTy::Minus),
60 (b':', TokenTy::Colon),
61 (b'!', TokenTy::Bang),
62 (b'=', TokenTy::Eq),
63 (b'&', TokenTy::And),
64 (b'|', TokenTy::Or),
65 (b'/', TokenTy::Div),
66 (b'+', TokenTy::Plus),
67 (b'^', TokenTy::Xor),
68 (b'*', TokenTy::Star),
69 (b'%', TokenTy::Mod),
70 (b'.', TokenTy::Dot),
71 (b'~', TokenTy::Tilde),
72];
73
74pub fn try_consume_trivial_token(lexer: &mut Lexer) -> Option<Token> {
78 let bytes_remaining: usize = lexer.bytes_remaining();
80
81 if bytes_remaining == 0 {
83 return None;
84 }
85
86 if let Some(bytes) = lexer.remaining.as_str().as_bytes().get(0..3) {
88 for (pattern, kind) in THREE_ASCII_TRIVIAL_TOKENS {
89 if bytes == *pattern {
90 return Some(lexer.split_token_unchecked(3, *kind));
92 }
93 }
94 }
95
96 if bytes_remaining >= 2 {
99 let bytes: &[u8] = unsafe { lexer.remaining.as_str().as_bytes().get_unchecked(0..2) };
102
103 for (pattern, kind) in TWO_ASCII_TRIVIAL_TOKENS {
105 if bytes == *pattern {
106 return Some(lexer.split_token_unchecked(2, *kind));
109 }
110 }
111 }
112
113 let byte: &u8 = unsafe { lexer.remaining.as_str().as_bytes().get_unchecked(0) };
116
117 for (pattern, kind) in SINGLE_ASCII_CHAR_TRIVIAL_TOKENS {
118 if byte == pattern {
119 return Some(lexer.split_token_unchecked(1, *kind));
122 }
123 }
124
125 None
127}
128
129#[cfg(test)]
130mod tests {
131 use super::{Lexer, TokenTy};
132
133 #[test]
134 fn plus_and_plus_eq_tokens() {
135 let mut plus = Lexer::new_test("+");
136 let mut plus_eq = Lexer::new_test("+=");
137
138 let plus_token = plus.next_token().unwrap();
139 let plus_eq_token = plus_eq.next_token().unwrap();
140
141 assert_eq!(plus.bytes_remaining(), 0);
142 assert_eq!(plus_eq.bytes_remaining(), 0);
143 assert_eq!(plus_token.variant, TokenTy::Plus);
144 assert_eq!(plus_eq_token.variant, TokenTy::PlusEq);
145 }
146
147 #[test]
148 fn plus_one_token() {
149 let mut plus_one = Lexer::new_test("+1");
150 let plus_token = plus_one.next_token().unwrap();
151 assert_eq!(plus_one.bytes_remaining(), 1);
152 assert_eq!(plus_token.variant, TokenTy::Plus);
153 assert_eq!(plus_token.fragment.len(), 1);
154 }
155
156 #[test]
157 fn greedy_multi_char_tokens() {
158 let cases = [
159 ("..=", TokenTy::DotDotEq),
160 ("..", TokenTy::DotDot),
161 (".", TokenTy::Dot),
162 ("<<=", TokenTy::LtLtEq),
163 (">>=", TokenTy::GtGtEq),
164 (">>", TokenTy::GtGt),
165 ("~", TokenTy::Tilde),
166 ("->", TokenTy::SingleArrow),
167 ];
168
169 for (source, expected) in cases {
170 let mut lexer = Lexer::new_test(source);
171 let token = lexer.next_token().unwrap();
172 assert_eq!(token.variant, expected, "lexing {source:?}");
173 assert_eq!(lexer.bytes_remaining(), 0, "lexing {source:?}");
174 }
175 }
176}