Skip to main content

wright/lexer/
integer_literal.rs

1//! Implementation for lexing integer and float literals.
2
3use super::{
4    Lexer,
5    token::{Token, TokenTy},
6};
7
8/// Count the bytes at the start of `bytes` that are digits in the given radix or underscores.
9fn digits_and_underscores(bytes: &[u8], radix: u32) -> usize {
10    bytes
11        .iter()
12        .take_while(|b| **b == b'_' || (**b as char).is_digit(radix))
13        .count()
14}
15
16/// Attempt to lex and consume a [TokenTy::IntegerLiteral] or [TokenTy::FloatLiteral] from the lexer.
17///
18/// Integer literals may have a radix prefix (`0x`, `0X`, `0o`, `0b`, `0B`) and may contain underscores after their
19/// first digit. Float literals are always decimal, and have a fractional part (`1.5`), an exponent (`1e10`), or both
20/// (`1.5e-3`). A `.` is only considered part of a float if it is followed by a decimal digit, so `1..2` and `1.foo`
21/// lex the way you would expect.
22pub fn try_consume_number_literal(lexer: &mut Lexer) -> Option<Token> {
23    // All the characters in a number literal are ASCII, so work with bytes directly.
24    let bytes: &[u8] = lexer.remaining.as_str().as_bytes();
25
26    // We can only continue lexing if the first byte exists and is an ascii decimal digit.
27    if !bytes.first()?.is_ascii_digit() {
28        return None;
29    }
30
31    // Handle radix prefixes.
32    let radix: Option<u32> = match bytes {
33        [b'0', b'x' | b'X', ..] => Some(16),
34        [b'0', b'b' | b'B', ..] => Some(2),
35        [b'0', b'o', ..] => Some(8),
36        _ => None,
37    };
38
39    if let Some(radix) = radix {
40        // The first character after the prefix is required to be a digit, not underscore.
41        if !bytes.get(2).is_some_and(|b| (*b as char).is_digit(radix)) {
42            return None;
43        }
44
45        let len = 2 + digits_and_underscores(&bytes[2..], radix);
46        return Some(lexer.split_token(len, TokenTy::IntegerLiteral));
47    }
48
49    // Decimal integer part.
50    let mut len = digits_and_underscores(bytes, 10);
51    let mut is_float = false;
52
53    // Fractional part -- only if the dot is followed by a digit.
54    if bytes.get(len) == Some(&b'.') && bytes.get(len + 1).is_some_and(u8::is_ascii_digit) {
55        is_float = true;
56        len += 1 + digits_and_underscores(&bytes[len + 1..], 10);
57    }
58
59    // Exponent -- only if it's followed by (an optionally signed) digit.
60    if matches!(bytes.get(len), Some(b'e' | b'E')) {
61        let sign_len = usize::from(matches!(bytes.get(len + 1), Some(b'+' | b'-')));
62
63        if bytes
64            .get(len + 1 + sign_len)
65            .is_some_and(u8::is_ascii_digit)
66        {
67            is_float = true;
68            len += 1 + sign_len + digits_and_underscores(&bytes[len + 1 + sign_len..], 10);
69        }
70    }
71
72    let variant = if is_float {
73        TokenTy::FloatLiteral
74    } else {
75        TokenTy::IntegerLiteral
76    };
77
78    Some(lexer.split_token(len, variant))
79}
80
81#[cfg(test)]
82mod tests {
83    use super::try_consume_number_literal;
84    use super::{Lexer, TokenTy};
85
86    fn lex_one(source: &'static str) -> Option<(TokenTy, String)> {
87        let mut lexer = Lexer::new_test(source);
88        try_consume_number_literal(&mut lexer).map(|t| (t.variant, t.fragment.as_str().to_owned()))
89    }
90
91    #[test]
92    fn integer_literal() {
93        let mut lexer = Lexer::new_test("123_456_789.");
94
95        let token = lexer.next_token().unwrap();
96
97        assert_eq!(token.fragment.as_str(), "123_456_789");
98        assert_eq!(token.variant, TokenTy::IntegerLiteral);
99        assert_eq!(lexer.remaining.as_str(), ".");
100    }
101
102    #[test]
103    fn cant_start_with_underscore() {
104        let mut lexer = Lexer::new_test("0x__10");
105
106        assert!(try_consume_number_literal(&mut lexer).is_none());
107
108        assert_eq!(lexer.remaining.as_str(), "0x__10");
109    }
110
111    #[test]
112    fn radix_prefixes() {
113        assert_eq!(lex_one("0xFF_ff"), Some((TokenTy::IntegerLiteral, "0xFF_ff".into())));
114        assert_eq!(lex_one("0b1012"), Some((TokenTy::IntegerLiteral, "0b101".into())));
115        assert_eq!(lex_one("0o777"), Some((TokenTy::IntegerLiteral, "0o777".into())));
116        assert_eq!(lex_one("0x1.5"), Some((TokenTy::IntegerLiteral, "0x1".into())));
117    }
118
119    #[test]
120    fn floats() {
121        assert_eq!(lex_one("1.5"), Some((TokenTy::FloatLiteral, "1.5".into())));
122        assert_eq!(lex_one("1_000.000_1"), Some((TokenTy::FloatLiteral, "1_000.000_1".into())));
123        assert_eq!(lex_one("1e10"), Some((TokenTy::FloatLiteral, "1e10".into())));
124        assert_eq!(lex_one("2.5E-3"), Some((TokenTy::FloatLiteral, "2.5E-3".into())));
125        assert_eq!(lex_one("2.5e+3x"), Some((TokenTy::FloatLiteral, "2.5e+3".into())));
126    }
127
128    #[test]
129    fn not_floats() {
130        assert_eq!(lex_one("1..2"), Some((TokenTy::IntegerLiteral, "1".into())));
131        assert_eq!(lex_one("1.foo"), Some((TokenTy::IntegerLiteral, "1".into())));
132        assert_eq!(lex_one("1."), Some((TokenTy::IntegerLiteral, "1".into())));
133        assert_eq!(lex_one("1e"), Some((TokenTy::IntegerLiteral, "1".into())));
134        assert_eq!(lex_one("1e+"), Some((TokenTy::IntegerLiteral, "1".into())));
135    }
136}