wright/lexer/
integer_literal.rs1use super::{
4 Lexer,
5 token::{Token, TokenTy},
6};
7
8fn digits_and_underscores(bytes: &[u8], radix: u32) -> usize {
10 bytes
11 .iter()
12 .take_while(|b| **b == b'_' || (**b as char).is_digit(radix))
13 .count()
14}
15
16pub fn try_consume_number_literal(lexer: &mut Lexer) -> Option<Token> {
23 let bytes: &[u8] = lexer.remaining.as_str().as_bytes();
25
26 if !bytes.first()?.is_ascii_digit() {
28 return None;
29 }
30
31 let radix: Option<u32> = match bytes {
33 [b'0', b'x' | b'X', ..] => Some(16),
34 [b'0', b'b' | b'B', ..] => Some(2),
35 [b'0', b'o', ..] => Some(8),
36 _ => None,
37 };
38
39 if let Some(radix) = radix {
40 if !bytes.get(2).is_some_and(|b| (*b as char).is_digit(radix)) {
42 return None;
43 }
44
45 let len = 2 + digits_and_underscores(&bytes[2..], radix);
46 return Some(lexer.split_token(len, TokenTy::IntegerLiteral));
47 }
48
49 let mut len = digits_and_underscores(bytes, 10);
51 let mut is_float = false;
52
53 if bytes.get(len) == Some(&b'.') && bytes.get(len + 1).is_some_and(u8::is_ascii_digit) {
55 is_float = true;
56 len += 1 + digits_and_underscores(&bytes[len + 1..], 10);
57 }
58
59 if matches!(bytes.get(len), Some(b'e' | b'E')) {
61 let sign_len = usize::from(matches!(bytes.get(len + 1), Some(b'+' | b'-')));
62
63 if bytes
64 .get(len + 1 + sign_len)
65 .is_some_and(u8::is_ascii_digit)
66 {
67 is_float = true;
68 len += 1 + sign_len + digits_and_underscores(&bytes[len + 1 + sign_len..], 10);
69 }
70 }
71
72 let variant = if is_float {
73 TokenTy::FloatLiteral
74 } else {
75 TokenTy::IntegerLiteral
76 };
77
78 Some(lexer.split_token(len, variant))
79}
80
81#[cfg(test)]
82mod tests {
83 use super::try_consume_number_literal;
84 use super::{Lexer, TokenTy};
85
86 fn lex_one(source: &'static str) -> Option<(TokenTy, String)> {
87 let mut lexer = Lexer::new_test(source);
88 try_consume_number_literal(&mut lexer).map(|t| (t.variant, t.fragment.as_str().to_owned()))
89 }
90
91 #[test]
92 fn integer_literal() {
93 let mut lexer = Lexer::new_test("123_456_789.");
94
95 let token = lexer.next_token().unwrap();
96
97 assert_eq!(token.fragment.as_str(), "123_456_789");
98 assert_eq!(token.variant, TokenTy::IntegerLiteral);
99 assert_eq!(lexer.remaining.as_str(), ".");
100 }
101
102 #[test]
103 fn cant_start_with_underscore() {
104 let mut lexer = Lexer::new_test("0x__10");
105
106 assert!(try_consume_number_literal(&mut lexer).is_none());
107
108 assert_eq!(lexer.remaining.as_str(), "0x__10");
109 }
110
111 #[test]
112 fn radix_prefixes() {
113 assert_eq!(lex_one("0xFF_ff"), Some((TokenTy::IntegerLiteral, "0xFF_ff".into())));
114 assert_eq!(lex_one("0b1012"), Some((TokenTy::IntegerLiteral, "0b101".into())));
115 assert_eq!(lex_one("0o777"), Some((TokenTy::IntegerLiteral, "0o777".into())));
116 assert_eq!(lex_one("0x1.5"), Some((TokenTy::IntegerLiteral, "0x1".into())));
117 }
118
119 #[test]
120 fn floats() {
121 assert_eq!(lex_one("1.5"), Some((TokenTy::FloatLiteral, "1.5".into())));
122 assert_eq!(lex_one("1_000.000_1"), Some((TokenTy::FloatLiteral, "1_000.000_1".into())));
123 assert_eq!(lex_one("1e10"), Some((TokenTy::FloatLiteral, "1e10".into())));
124 assert_eq!(lex_one("2.5E-3"), Some((TokenTy::FloatLiteral, "2.5E-3".into())));
125 assert_eq!(lex_one("2.5e+3x"), Some((TokenTy::FloatLiteral, "2.5e+3".into())));
126 }
127
128 #[test]
129 fn not_floats() {
130 assert_eq!(lex_one("1..2"), Some((TokenTy::IntegerLiteral, "1".into())));
131 assert_eq!(lex_one("1.foo"), Some((TokenTy::IntegerLiteral, "1".into())));
132 assert_eq!(lex_one("1."), Some((TokenTy::IntegerLiteral, "1".into())));
133 assert_eq!(lex_one("1e"), Some((TokenTy::IntegerLiteral, "1".into())));
134 assert_eq!(lex_one("1e+"), Some((TokenTy::IntegerLiteral, "1".into())));
135 }
136}