wright/lexer/
identifier.rs1use super::{Lexer, token::Token, token::TokenTy};
4use crate::source_tracking::fragment::Fragment;
5use std::str::Chars;
6use unicode_ident::{is_xid_continue, is_xid_start};
7
8fn identifier_or_keyword(fragment: Fragment) -> TokenTy {
11 use TokenTy::*;
12
13 match fragment.as_str() {
14 "record" => KwRecord,
15 "type" => KwType,
16 "enum" => KwEnum,
17 "union" => KwUnion,
18 "func" => KwFunc,
19 "pure" => KwPure,
20 "unsafe" => KwUnsafe,
21 "naked" => KwNaked,
22 "repr" => KwRepr,
23 "impl" => KwImpl,
24 "constrain" => KwConstrain,
25 "constraint" => KwConstraint,
26 "references" => KwReferences,
27 "trait" => KwTrait,
28 "const" => KwConst,
29 "where" => KwWhere,
30 "return" => KwReturn,
31 "break" => KwBreak,
32 "continue" => KwContinue,
33
34 "use" => KwUse,
35 "as" => KwAs,
36 "mod" => KwMod,
37 "pub" => KwPub,
38
39 "if" => KwIf,
40 "else" => KwElse,
41 "match" => KwMatch,
42
43 "for" => KwFor,
44 "in" => KwIn,
45 "while" => KwWhile,
46 "loop" => KwLoop,
47
48 "let" => KwLet,
49 "var" => KwVar,
50
51 "true" => KwTrue,
52 "false" => KwFalse,
53
54 "bool" => KwBool,
55 "u8" => KwU8,
56 "i8" => KwI8,
57 "u16" => KwU16,
58 "i16" => KwI16,
59 "u32" => KwU32,
60 "i32" => KwI32,
61 "f32" => KwF32,
62 "u64" => KwU64,
63 "i64" => KwI64,
64 "f64" => KwF64,
65 "char" => KwChar,
66
67 "_" => Underscore,
68
69 _ => Identifier,
70 }
71}
72
73pub fn try_consume_keyword_or_identifier(lexer: &mut Lexer) -> Option<Token> {
75 let mut chars: Chars = lexer.remaining.chars();
77 let next: char = chars.next().filter(|c| is_xid_start(*c) || *c == '_')?;
81 let mut bytes_consumed: usize = next.len_utf8();
83
84 bytes_consumed += chars
86 .take_while(|c| is_xid_continue(*c))
87 .map(char::len_utf8)
88 .sum::<usize>();
89
90 let (token_fragment, new_remaining): (Fragment, Fragment) =
94 lexer.remaining.split_at_unchecked(bytes_consumed);
95
96 let variant: TokenTy = identifier_or_keyword(token_fragment.clone());
98
99 lexer.remaining = new_remaining;
101
102 Some(Token {
104 variant,
105 fragment: token_fragment,
106 })
107}
108
109#[cfg(test)]
110mod tests {
111 use super::{Lexer, TokenTy};
112
113 #[test]
114 fn identifiers_and_keywords() {
115 let mut lexer = Lexer::new_test("const TEST");
116
117 assert_eq!(lexer.next_token().unwrap().variant, TokenTy::KwConst);
118 assert_eq!(lexer.next_token().unwrap().variant, TokenTy::Whitespace);
119 assert_eq!(lexer.next_token().unwrap().variant, TokenTy::Identifier);
120 }
121}