wright/lexer/quoted.rs
1//! Lexing implementation for quoted literals.
2
3use super::{Lexer, token::Token, token::TokenTy};
4use std::str::Chars;
5
6/// Attempt to parse a quoted literal. This includes [TokenTy::StringLiteral], [TokenTy::CharLiteral], and
7/// [TokenTy::FormatStringLiteral] (and their unterminated counterparts, such as
8/// [TokenTy::UnterminatedStringLiteral]).
9pub fn try_consume_quoted_literal(lexer: &mut Lexer) -> Option<Token> {
10 // Make a chars iterator to lex from.
11 let mut chars: Chars = lexer.remaining.chars();
12 // Get the first char from the character iterator.
13 // Return none if the first character doesn't exist or is not one of the quote terminating characters.
14 let first: char = chars.next().filter(|c| ['\'', '"', '`'].contains(c))?;
15 // Track number of bytes consumed.
16 let mut bytes_consumed: usize = first.len_utf8();
17 // Track whether the quoted literal is terminated.
18 let mut is_terminated: bool = false;
19
20 // Consume from the iterator while possible.
21 while let Some(consumed) = chars.next() {
22 // Update the number of bytes consumed.
23 bytes_consumed += consumed.len_utf8();
24
25 // Check if the character matches the starting char.
26 // If so, record the literal as terminated and break this loop.
27 if consumed == first {
28 is_terminated = true;
29 break;
30 }
31
32 // If the character we just consumed is a backslash.
33 // We only handle escaped terminators here, rather than parsing actual meaning.
34 // Consume the next character if there is one, regardless of what it is.
35 // This prevents an escaped terminator from ending the literal.
36 if consumed == '\\' {
37 // If there is no next char, do not add anything to the number of bytes consumed.
38 bytes_consumed += chars.next().map(char::len_utf8).unwrap_or(0);
39 }
40 }
41
42 // Return when we have either reached a terminator or run out of characters.
43 // First determine the variant to return.
44 let variant: TokenTy = match (first, is_terminated) {
45 ('\'', true) => TokenTy::CharLiteral,
46 ('\'', false) => TokenTy::UnterminatedCharLiteral,
47 ('"', true) => TokenTy::StringLiteral,
48 ('"', false) => TokenTy::UnterminatedStringLiteral,
49 ('`', true) => TokenTy::FormatStringLiteral,
50 ('`', false) => TokenTy::UnterminatedFormatStringLiteral,
51 _ => unreachable!("There are no other quoted literals"),
52 };
53
54 // Summing char lengths from the iterator should never give us an invalid or out of bounds index.
55 Some(lexer.split_token_unchecked(bytes_consumed, variant))
56}
57
58#[cfg(test)]
59mod tests {
60 use super::super::{Lexer, token::TokenTy};
61
62 #[test]
63 fn string_literal() {
64 let mut lexer = Lexer::new_test(r#""Test string literal""#);
65 let token = lexer.next_token().unwrap();
66 assert_eq!(token.variant, TokenTy::StringLiteral);
67 assert_eq!(token.fragment.as_str(), "\"Test string literal\"");
68 }
69}