Skip to main content

wright/lexer/
quoted.rs

1//! Lexing implementation for quoted literals.
2
3use super::{Lexer, token::Token, token::TokenTy};
4use std::str::Chars;
5
6/// Attempt to parse a quoted literal. This includes [TokenTy::StringLiteral], [TokenTy::CharLiteral], and
7/// [TokenTy::FormatStringLiteral] (and their unterminated counterparts, such as
8/// [TokenTy::UnterminatedStringLiteral]).
9pub fn try_consume_quoted_literal(lexer: &mut Lexer) -> Option<Token> {
10    // Make a chars iterator to lex from.
11    let mut chars: Chars = lexer.remaining.chars();
12    // Get the first char from the character iterator.
13    // Return none if the first character doesn't exist or is not one of the quote terminating characters.
14    let first: char = chars.next().filter(|c| ['\'', '"', '`'].contains(c))?;
15    // Track number of bytes consumed.
16    let mut bytes_consumed: usize = first.len_utf8();
17    // Track whether the quoted literal is terminated.
18    let mut is_terminated: bool = false;
19
20    // Consume from the iterator while possible.
21    while let Some(consumed) = chars.next() {
22        // Update the number of bytes consumed.
23        bytes_consumed += consumed.len_utf8();
24
25        // Check if the character matches the starting char.
26        // If so, record the literal as terminated and break this loop.
27        if consumed == first {
28            is_terminated = true;
29            break;
30        }
31
32        // If the character we just consumed is a backslash.
33        // We only handle escaped terminators here, rather than parsing actual meaning.
34        // Consume the next character if there is one, regardless of what it is.
35        // This prevents an escaped terminator from ending the literal.
36        if consumed == '\\' {
37            // If there is no next char, do not add anything to the number of bytes consumed.
38            bytes_consumed += chars.next().map(char::len_utf8).unwrap_or(0);
39        }
40    }
41
42    // Return when we have either reached a terminator or run out of characters.
43    // First determine the variant to return.
44    let variant: TokenTy = match (first, is_terminated) {
45        ('\'', true) => TokenTy::CharLiteral,
46        ('\'', false) => TokenTy::UnterminatedCharLiteral,
47        ('"', true) => TokenTy::StringLiteral,
48        ('"', false) => TokenTy::UnterminatedStringLiteral,
49        ('`', true) => TokenTy::FormatStringLiteral,
50        ('`', false) => TokenTy::UnterminatedFormatStringLiteral,
51        _ => unreachable!("There are no other quoted literals"),
52    };
53
54    // Summing char lengths from the iterator should never give us an invalid or out of bounds index.
55    Some(lexer.split_token_unchecked(bytes_consumed, variant))
56}
57
58#[cfg(test)]
59mod tests {
60    use super::super::{Lexer, token::TokenTy};
61
62    #[test]
63    fn string_literal() {
64        let mut lexer = Lexer::new_test(r#""Test string literal""#);
65        let token = lexer.next_token().unwrap();
66        assert_eq!(token.variant, TokenTy::StringLiteral);
67        assert_eq!(token.fragment.as_str(), "\"Test string literal\"");
68    }
69}