wright/lexer.rs
1//! First pass lexer that gets run on the source code and returns a series of tokens with their associated [Fragment]s.
2//!
3//! Every byte of the source ends up in exactly one token -- whitespace and comments are produced as tokens too
4//! (see [TokenTy::is_trivia]), which lets the parser build a lossless syntax tree.
5
6use self::comments::{try_match_block_comment, try_match_single_line_comment};
7use self::integer_literal::try_consume_number_literal;
8use self::quoted::try_consume_quoted_literal;
9use crate::source_tracking::SourceRef;
10use crate::source_tracking::fragment::Fragment;
11use token::{Token, TokenTy};
12
13pub mod comments;
14pub mod identifier;
15pub mod integer_literal;
16pub mod quoted;
17pub mod token;
18pub mod trivial;
19
20/// The lexical analyser for wright. This produces a series of tokens that make up the larger elements of the language.
21#[derive(Debug, Clone)]
22pub struct Lexer {
23 /// The remaining source code that has not been processed and returned as a token from the iterator yet.
24 pub remaining: Fragment,
25}
26
27impl Lexer {
28 /// Get the number of bytes remaining that we need to transform into tokens.
29 pub const fn bytes_remaining(&self) -> usize {
30 self.remaining.len()
31 }
32
33 /// Construct a new [Lexer] over a given source reference.
34 pub fn new(source: SourceRef) -> Self {
35 Lexer {
36 remaining: source.as_fragment(),
37 }
38 }
39
40 /// Available in test cases, creates a new [Lexer] over a given static [str]ing.
41 ///
42 /// The instantiated [Source] in this [Lexer] has its name set to [FileName::None].
43 ///
44 /// This function is limited to this crate because `#[cfg(test)]` items are not available
45 /// externally, however it should be relatively easy to reproduce.
46 ///
47 /// [Source]: crate::source_tracking::source::Source
48 /// [FileName::None]: crate::source_tracking::filename::FileName::None
49 #[cfg(test)]
50 pub(crate) fn new_test(source: &'static str) -> Self {
51 use crate::source_tracking::{filename::FileName, source::Source};
52 use std::sync::Arc;
53
54 Lexer {
55 remaining: Fragment {
56 source: Arc::new(Source::new_from_static_str(FileName::None, source)),
57 range: 0..source.len(),
58 },
59 }
60 }
61
62 /// Make a token by splitting a given number of bytes off of [Lexer::remaining]
63 /// and labeling them with the given kind.
64 ///
65 /// # Panics:
66 /// - Panics if the number of bytes lands out of bounds or in the middle of a character.
67 fn split_token(&mut self, bytes: usize, kind: TokenTy) -> Token {
68 let (token_fragment, new_remaining_fragment) = self.remaining.split_at(bytes);
69 self.remaining = new_remaining_fragment;
70
71 Token {
72 variant: kind,
73 fragment: token_fragment,
74 }
75 }
76
77 /// Unchecked version of [Lexer::split_token].
78 ///
79 /// # Panics
80 /// - This function has the same potential to cause logic bugs and panics as [Fragment::split_at_unchecked].
81 fn split_token_unchecked(&mut self, bytes: usize, kind: TokenTy) -> Token {
82 let (token_fragment, new_remaining_fragment) = self.remaining.split_at_unchecked(bytes);
83 self.remaining = new_remaining_fragment;
84
85 Token {
86 variant: kind,
87 fragment: token_fragment,
88 }
89 }
90
91 /// "Fork" this lexer, creating a new [`Lexer`] at the same position as this one that can be used for
92 /// failable parsing. This can be compared to the original lexer it was forked from using [Lexer::offset_from].
93 pub fn fork(&self) -> Self {
94 self.clone()
95 }
96
97 /// Get the number of bytes between the origin's [remaining](Lexer::remaining) and
98 /// this [Lexer]'s [remaining](Lexer::remaining) using [`Fragment::offset_from`].
99 ///
100 /// # Panics
101 /// - This function panics under the same conditions as [`Fragment::offset_from`].
102 /// - Generally the best way to avoid panics is to only call this function on
103 /// [Lexer]s created using [Lexer::fork] on the `origin` lexer.
104 pub fn offset_from(&self, origin: &Self) -> usize {
105 self.remaining.offset_from(&origin.remaining)
106 }
107
108 /// Check if a pattern matches at the start of the [Lexer::remaining] [Fragment].
109 pub fn matches(&self, pattern: &str) -> bool {
110 self.remaining.as_str().starts_with(pattern)
111 }
112
113 /// Get the next token from the lexer.
114 pub fn next_token(&mut self) -> Option<Token> {
115 // If the remaining input is empty, there is no token.
116 if self.remaining.is_empty() {
117 return None;
118 }
119
120 // If there is whitespace, it becomes its own token.
121 // Use a little unsafe here since this check is done every time and needs to be fast.
122 {
123 let remaining_str = self.remaining.as_str();
124 let trimmed = remaining_str.trim_start().as_ptr();
125
126 // Calculate the delta by pointer offset.
127 // SAFETY: In this case, all the requirements of pointer::offset_from are satisfied.
128 let delta = unsafe { trimmed.offset_from(remaining_str.as_ptr()) };
129
130 if delta > 0 {
131 // trim_start should always return a valid string, and delta is just checked to be > 0.
132 return Some(self.split_token_unchecked(delta as usize, TokenTy::Whitespace));
133 }
134 }
135
136 // Attempt to parse a single line comment and then attempt a multi-line comment.
137 // Comments are emitted as tokens so that the parser can produce a lossless syntax tree.
138 if let Some((bytes, variant)) =
139 try_match_single_line_comment(self).or_else(|| try_match_block_comment(self))
140 {
141 return Some(self.split_token(bytes, variant));
142 }
143
144 // Handle a trivial token if there is one.
145 if let Some(token) = trivial::try_consume_trivial_token(self) {
146 return Some(token);
147 }
148
149 // Next attempt to match a keyword or identifier.
150 if let Some(token) = identifier::try_consume_keyword_or_identifier(self) {
151 return Some(token);
152 }
153
154 // Next attempt to parse an integer or float literal.
155 if let Some(number_lit) = try_consume_number_literal(self) {
156 return Some(number_lit);
157 }
158
159 // Next attempt to parse a quoted literal.
160 if let Some(quoted_lit) = try_consume_quoted_literal(self) {
161 return Some(quoted_lit);
162 }
163
164 // If we haven't matched at this point, produce a token marked as "Unknown".
165 // The unsafe is fine -- we know from above that there are remaining characters.
166 let unknown_char = unsafe { self.remaining.chars().next().unwrap_unchecked() };
167 Some(self.split_token(unknown_char.len_utf8(), TokenTy::Unknown))
168 }
169}
170
171impl Iterator for Lexer {
172 type Item = Token;
173
174 fn next(&mut self) -> Option<Self::Item> {
175 self.next_token()
176 }
177}