Skip to main content

wright/
lexer.rs

1//! First pass lexer that gets run on the source code and returns a series of tokens with their associated [Fragment]s.
2//!
3//! Every byte of the source ends up in exactly one token -- whitespace and comments are produced as tokens too
4//! (see [TokenTy::is_trivia]), which lets the parser build a lossless syntax tree.
5
6use self::comments::{try_match_block_comment, try_match_single_line_comment};
7use self::integer_literal::try_consume_number_literal;
8use self::quoted::try_consume_quoted_literal;
9use crate::source_tracking::SourceRef;
10use crate::source_tracking::fragment::Fragment;
11use token::{Token, TokenTy};
12
13pub mod comments;
14pub mod identifier;
15pub mod integer_literal;
16pub mod quoted;
17pub mod token;
18pub mod trivial;
19
20/// The lexical analyser for wright. This produces a series of tokens that make up the larger elements of the language.
21#[derive(Debug, Clone)]
22pub struct Lexer {
23    /// The remaining source code that has not been processed and returned as a token from the iterator yet.
24    pub remaining: Fragment,
25}
26
27impl Lexer {
28    /// Get the number of bytes remaining that we need to transform into tokens.
29    pub const fn bytes_remaining(&self) -> usize {
30        self.remaining.len()
31    }
32
33    /// Construct a new [Lexer] over a given source reference.
34    pub fn new(source: SourceRef) -> Self {
35        Lexer {
36            remaining: source.as_fragment(),
37        }
38    }
39
40    /// Available in test cases, creates a new [Lexer] over a given static [str]ing.
41    ///
42    /// The instantiated [Source] in this [Lexer] has its name set to [FileName::None].
43    ///
44    /// This function is limited to this crate because `#[cfg(test)]` items are not available
45    /// externally, however it should be relatively easy to reproduce.
46    ///
47    /// [Source]: crate::source_tracking::source::Source
48    /// [FileName::None]: crate::source_tracking::filename::FileName::None
49    #[cfg(test)]
50    pub(crate) fn new_test(source: &'static str) -> Self {
51        use crate::source_tracking::{filename::FileName, source::Source};
52        use std::sync::Arc;
53
54        Lexer {
55            remaining: Fragment {
56                source: Arc::new(Source::new_from_static_str(FileName::None, source)),
57                range: 0..source.len(),
58            },
59        }
60    }
61
62    /// Make a token by splitting a given number of bytes off of [Lexer::remaining]
63    /// and labeling them with the given kind.
64    ///
65    /// # Panics:
66    /// - Panics if the number of bytes lands out of bounds or in the middle of a character.
67    fn split_token(&mut self, bytes: usize, kind: TokenTy) -> Token {
68        let (token_fragment, new_remaining_fragment) = self.remaining.split_at(bytes);
69        self.remaining = new_remaining_fragment;
70
71        Token {
72            variant: kind,
73            fragment: token_fragment,
74        }
75    }
76
77    /// Unchecked version of [Lexer::split_token].
78    ///
79    /// # Panics
80    /// - This function has the same potential to cause logic bugs and panics as [Fragment::split_at_unchecked].
81    fn split_token_unchecked(&mut self, bytes: usize, kind: TokenTy) -> Token {
82        let (token_fragment, new_remaining_fragment) = self.remaining.split_at_unchecked(bytes);
83        self.remaining = new_remaining_fragment;
84
85        Token {
86            variant: kind,
87            fragment: token_fragment,
88        }
89    }
90
91    /// "Fork" this lexer, creating a new [`Lexer`] at the same position as this one that can be used for
92    /// failable parsing. This can be compared to the original lexer it was forked from using [Lexer::offset_from].
93    pub fn fork(&self) -> Self {
94        self.clone()
95    }
96
97    /// Get the number of bytes between the origin's [remaining](Lexer::remaining) and
98    /// this [Lexer]'s [remaining](Lexer::remaining) using [`Fragment::offset_from`].
99    ///
100    /// # Panics
101    /// - This function panics under the same conditions as [`Fragment::offset_from`].
102    /// - Generally the best way to avoid panics is to only call this function on
103    ///   [Lexer]s created using [Lexer::fork] on the `origin` lexer.
104    pub fn offset_from(&self, origin: &Self) -> usize {
105        self.remaining.offset_from(&origin.remaining)
106    }
107
108    /// Check if a pattern matches at the start of the [Lexer::remaining] [Fragment].
109    pub fn matches(&self, pattern: &str) -> bool {
110        self.remaining.as_str().starts_with(pattern)
111    }
112
113    /// Get the next token from the lexer.
114    pub fn next_token(&mut self) -> Option<Token> {
115        // If the remaining input is empty, there is no token.
116        if self.remaining.is_empty() {
117            return None;
118        }
119
120        // If there is whitespace, it becomes its own token.
121        // Use a little unsafe here since this check is done every time and needs to be fast.
122        {
123            let remaining_str = self.remaining.as_str();
124            let trimmed = remaining_str.trim_start().as_ptr();
125
126            // Calculate the delta by pointer offset.
127            // SAFETY: In this case, all the requirements of pointer::offset_from are satisfied.
128            let delta = unsafe { trimmed.offset_from(remaining_str.as_ptr()) };
129
130            if delta > 0 {
131                // trim_start should always return a valid string, and delta is just checked to be > 0.
132                return Some(self.split_token_unchecked(delta as usize, TokenTy::Whitespace));
133            }
134        }
135
136        // Attempt to parse a single line comment and then attempt a multi-line comment.
137        // Comments are emitted as tokens so that the parser can produce a lossless syntax tree.
138        if let Some((bytes, variant)) =
139            try_match_single_line_comment(self).or_else(|| try_match_block_comment(self))
140        {
141            return Some(self.split_token(bytes, variant));
142        }
143
144        // Handle a trivial token if there is one.
145        if let Some(token) = trivial::try_consume_trivial_token(self) {
146            return Some(token);
147        }
148
149        // Next attempt to match a keyword or identifier.
150        if let Some(token) = identifier::try_consume_keyword_or_identifier(self) {
151            return Some(token);
152        }
153
154        // Next attempt to parse an integer or float literal.
155        if let Some(number_lit) = try_consume_number_literal(self) {
156            return Some(number_lit);
157        }
158
159        // Next attempt to parse a quoted literal.
160        if let Some(quoted_lit) = try_consume_quoted_literal(self) {
161            return Some(quoted_lit);
162        }
163
164        // If we haven't matched at this point, produce a token marked as "Unknown".
165        // The unsafe is fine -- we know from above that there are remaining characters.
166        let unknown_char = unsafe { self.remaining.chars().next().unwrap_unchecked() };
167        Some(self.split_token(unknown_char.len_utf8(), TokenTy::Unknown))
168    }
169}
170
171impl Iterator for Lexer {
172    type Item = Token;
173
174    fn next(&mut self) -> Option<Self::Item> {
175        self.next_token()
176    }
177}