Skip to main content

wright/lexer/
comments.rs

1//! Implementation of comment token lexing.
2//!
3//! Comments are produced as tokens (rather than being discarded) so that the syntax tree produced by the parser is
4//! lossless. The parser skips over them.
5
6use super::{Lexer, token::TokenTy};
7
8/// The pattern that begins any single line comments (including doc comments).
9pub const SINGLE_LINE_COMMENT_PREFIX: &str = "//";
10
11/// The pattern that starts any multi-line comments (including doc comments).
12pub const MULTI_LINE_COMMENT_START: &str = "/*";
13
14/// The pattern that ends any multi-line comments (including doc comments).
15pub const MULTI_LINE_COMMENT_END: &str = "*/";
16
17/// Attempt to match a single line comment from the start of the [Lexer::remaining] fragment.
18///
19/// Returns [None] if there is no single line comment at the start of the remaining source. Otherwise returns the
20/// number of bytes in the comment (not including the line ending) and the kind of comment:
21/// - `/// ...` is a [TokenTy::OuterDocComment] (documents the following item).
22/// - `//! ...` is a [TokenTy::InnerDocComment] (documents the enclosing item).
23/// - Anything else (including `//// ...`) is a plain [TokenTy::LineComment].
24///
25/// Generally I'm trying to follow the [rust comment spec] here.
26///
27/// [rust comment spec]: https://doc.rust-lang.org/reference/comments.html
28pub fn try_match_single_line_comment(lexer: &Lexer) -> Option<(usize, TokenTy)> {
29    let rest = lexer
30        .remaining
31        .as_str()
32        .strip_prefix(SINGLE_LINE_COMMENT_PREFIX)?;
33
34    let variant = if rest.starts_with('/') && !rest.starts_with("//") {
35        TokenTy::OuterDocComment
36    } else if rest.starts_with('!') {
37        TokenTy::InnerDocComment
38    } else {
39        TokenTy::LineComment
40    };
41
42    // Consume until a newline, carriage return, or the end of the source fragment.
43    let body_len = rest.find(['\r', '\n']).unwrap_or(rest.len());
44
45    Some((SINGLE_LINE_COMMENT_PREFIX.len() + body_len, variant))
46}
47
48/// Attempt to match a block comment from the start of the [Lexer::remaining] fragment.
49///
50/// Returns [None] if there is no block comment at the start of the remaining source. Otherwise returns the number of
51/// bytes in the comment and its kind:
52/// - `/** ... */` is a [TokenTy::OuterBlockDocComment].
53/// - `/*! ... */` is a [TokenTy::InnerBlockDocComment].
54/// - `/**/`, `/***/`, `/*** ... */`, and anything else are plain [TokenTy::BlockComment]s.
55/// - A comment that is not closed before the end of the source is a [TokenTy::UnterminatedBlockComment].
56///
57/// Block comments nest, so `/* /* */ */` is a single comment.
58pub fn try_match_block_comment(lexer: &Lexer) -> Option<(usize, TokenTy)> {
59    let source = lexer.remaining.as_str();
60
61    if !source.starts_with(MULTI_LINE_COMMENT_START) {
62        return None;
63    }
64
65    // Handle corner cases here so we don't have to below.
66    // These are both considered empty non-documenting comments.
67    if source.starts_with("/***/") {
68        return Some((5, TokenTy::BlockComment));
69    }
70
71    if source.starts_with("/**/") {
72        return Some((4, TokenTy::BlockComment));
73    }
74
75    let after_start = &source[MULTI_LINE_COMMENT_START.len()..];
76    let is_outer_doc = after_start.starts_with('*') && !after_start.starts_with("**");
77    let is_inner_doc = after_start.starts_with('!');
78
79    // Walk the bytes tracking nesting depth. All the delimiters are ASCII so byte-wise scanning is safe for UTF-8.
80    let bytes = source.as_bytes();
81    let mut depth: usize = 1;
82    let mut index: usize = MULTI_LINE_COMMENT_START.len();
83
84    while index < bytes.len() {
85        match (bytes[index], bytes.get(index + 1)) {
86            (b'/', Some(b'*')) => {
87                depth += 1;
88                index += 2;
89            }
90
91            (b'*', Some(b'/')) => {
92                depth -= 1;
93                index += 2;
94
95                if depth == 0 {
96                    let variant = if is_outer_doc {
97                        TokenTy::OuterBlockDocComment
98                    } else if is_inner_doc {
99                        TokenTy::InnerBlockDocComment
100                    } else {
101                        TokenTy::BlockComment
102                    };
103
104                    return Some((index, variant));
105                }
106            }
107
108            _ => index += 1,
109        }
110    }
111
112    // We ran out of source before closing the comment.
113    Some((source.len(), TokenTy::UnterminatedBlockComment))
114}
115
116#[cfg(test)]
117mod tests {
118    use super::Lexer;
119    use crate::lexer::token::TokenTy;
120
121    fn single(source: &'static str) -> (TokenTy, String) {
122        let mut lexer = Lexer::new_test(source);
123        let token = lexer.next_token().unwrap();
124        (token.variant, token.fragment.as_str().to_owned())
125    }
126
127    #[test]
128    fn single_line_comment() {
129        let mut lexer = Lexer::new_test("// test comment \nx");
130        let token = lexer.next_token().unwrap();
131        assert_eq!(token.variant, TokenTy::LineComment);
132        assert_eq!(token.fragment.as_str(), "// test comment ");
133        assert_eq!(lexer.next_token().unwrap().variant, TokenTy::Whitespace);
134        assert_eq!(lexer.next_token().unwrap().variant, TokenTy::Identifier);
135    }
136
137    #[test]
138    fn doc_comment_kinds() {
139        assert_eq!(single("/// outer").0, TokenTy::OuterDocComment);
140        assert_eq!(single("//! inner").0, TokenTy::InnerDocComment);
141        assert_eq!(single("//// not a doc").0, TokenTy::LineComment);
142        assert_eq!(single("/** outer */").0, TokenTy::OuterBlockDocComment);
143        assert_eq!(single("/*! inner */").0, TokenTy::InnerBlockDocComment);
144        assert_eq!(single("/*** not a doc */").0, TokenTy::BlockComment);
145        assert_eq!(single("/**/").0, TokenTy::BlockComment);
146        assert_eq!(single("/***/").0, TokenTy::BlockComment);
147    }
148
149    #[test]
150    fn nested_block_comment() {
151        let (variant, text) = single("/* a /* b */ c */ rest");
152        assert_eq!(variant, TokenTy::BlockComment);
153        assert_eq!(text, "/* a /* b */ c */");
154    }
155
156    #[test]
157    fn unterminated_block_comment() {
158        let (variant, text) = single("/* a /* b */ c");
159        assert_eq!(variant, TokenTy::UnterminatedBlockComment);
160        assert_eq!(text, "/* a /* b */ c");
161    }
162
163    #[test]
164    fn comment_with_unicode() {
165        let (variant, text) = single("/* ☃ — ✓ */x");
166        assert_eq!(variant, TokenTy::BlockComment);
167        assert_eq!(text, "/* ☃ — ✓ */");
168    }
169}