Skip to main content

wright/lexer/
identifier.rs

1//! Implementation related to parsing keywords and identifiers.
2
3use super::{Lexer, token::Token, token::TokenTy};
4use crate::source_tracking::fragment::Fragment;
5use std::str::Chars;
6use unicode_ident::{is_xid_continue, is_xid_start};
7
8/// Try to match a fragment recognized to be an identifier or keyword to
9/// a keyword or return [TokenTy::Identifier].
10fn identifier_or_keyword(fragment: Fragment) -> TokenTy {
11    use TokenTy::*;
12
13    match fragment.as_str() {
14        "record" => KwRecord,
15        "type" => KwType,
16        "enum" => KwEnum,
17        "union" => KwUnion,
18        "func" => KwFunc,
19        "pure" => KwPure,
20        "unsafe" => KwUnsafe,
21        "naked" => KwNaked,
22        "repr" => KwRepr,
23        "impl" => KwImpl,
24        "constrain" => KwConstrain,
25        "constraint" => KwConstraint,
26        "references" => KwReferences,
27        "trait" => KwTrait,
28        "const" => KwConst,
29        "where" => KwWhere,
30        "return" => KwReturn,
31        "break" => KwBreak,
32        "continue" => KwContinue,
33
34        "use" => KwUse,
35        "as" => KwAs,
36        "mod" => KwMod,
37        "pub" => KwPub,
38
39        "if" => KwIf,
40        "else" => KwElse,
41        "match" => KwMatch,
42
43        "for" => KwFor,
44        "in" => KwIn,
45        "while" => KwWhile,
46        "loop" => KwLoop,
47
48        "let" => KwLet,
49        "var" => KwVar,
50
51        "true" => KwTrue,
52        "false" => KwFalse,
53
54        "bool" => KwBool,
55        "u8" => KwU8,
56        "i8" => KwI8,
57        "u16" => KwU16,
58        "i16" => KwI16,
59        "u32" => KwU32,
60        "i32" => KwI32,
61        "f32" => KwF32,
62        "u64" => KwU64,
63        "i64" => KwI64,
64        "f64" => KwF64,
65        "char" => KwChar,
66
67        "_" => Underscore,
68
69        _ => Identifier,
70    }
71}
72
73/// Attempt to consume a keyword/[identifier](TokenTy::Identifier)/[underscore](TokenTy::Underscore) from the lexer.
74pub fn try_consume_keyword_or_identifier(lexer: &mut Lexer) -> Option<Token> {
75    // Get a character iterator that we can pull from.
76    let mut chars: Chars = lexer.remaining.chars();
77    // Get the next character from the iterator, consider it the first char of any potential match.
78    // Make sure it's a valid identifier start (includes start to all keywords) or is an underscore.
79    // If it does not exist or match predicates, return None.
80    let next: char = chars.next().filter(|c| is_xid_start(*c) || *c == '_')?;
81    // Store/track the number of bytes consumed so far.
82    let mut bytes_consumed: usize = next.len_utf8();
83
84    // Take remaining chars and add to sum.
85    bytes_consumed += chars
86        .take_while(|c| is_xid_continue(*c))
87        .map(char::len_utf8)
88        .sum::<usize>();
89
90    // Split the token and the new remaining fragment.
91    // VALIDITY: The character iterator should guarantee that we land on a valid character boundary within the bounds
92    // of the fragment.
93    let (token_fragment, new_remaining): (Fragment, Fragment) =
94        lexer.remaining.split_at_unchecked(bytes_consumed);
95
96    // Get the variant of token to produce.
97    let variant: TokenTy = identifier_or_keyword(token_fragment.clone());
98
99    // Update the lexer's remaining fragment.
100    lexer.remaining = new_remaining;
101
102    // Return the token.
103    Some(Token {
104        variant,
105        fragment: token_fragment,
106    })
107}
108
109#[cfg(test)]
110mod tests {
111    use super::{Lexer, TokenTy};
112
113    #[test]
114    fn identifiers_and_keywords() {
115        let mut lexer = Lexer::new_test("const TEST");
116
117        assert_eq!(lexer.next_token().unwrap().variant, TokenTy::KwConst);
118        assert_eq!(lexer.next_token().unwrap().variant, TokenTy::Whitespace);
119        assert_eq!(lexer.next_token().unwrap().variant, TokenTy::Identifier);
120    }
121}