nifty_syntax/
lexer.rs

1//! Turns source text into tokens. Strings are split into text and interpolation tokens so
2//! the parser handles `"{expr}"` with the ordinary expression grammar.
3
4use crate::diagnostic::Diagnostic;
5use crate::span::{FileId, Span};
6use crate::token::{Token, TokenKind};
7
8pub(crate) fn lex(source: &str, file: FileId) -> (Vec<Token>, Vec<Diagnostic>) {
9    let mut lexer = Lexer {
10        file,
11        source,
12        bytes: source.as_bytes(),
13        pos: 0,
14        modes: vec![Mode::Code { open_braces: 0 }],
15        tokens: Vec::new(),
16        diagnostics: Vec::new(),
17    };
18    lexer.run();
19    (lexer.tokens, lexer.diagnostics)
20}
21
22/// The text of a `StringText` token with its escapes applied.
23pub(crate) fn unescape(raw: &str) -> String {
24    let mut text = String::with_capacity(raw.len());
25    let mut chars = raw.chars();
26    while let Some(c) = chars.next() {
27        if c != '\\' {
28            text.push(c);
29            continue;
30        }
31        match chars.next() {
32            Some('n') => text.push('\n'),
33            Some('t') => text.push('\t'),
34            Some(escaped) => text.push(escaped),
35            None => {}
36        }
37    }
38    text
39}
40
41/// `text` written as the inside of a string literal, so [`unescape`] gives it back.
42pub(crate) fn escape(text: &str) -> String {
43    let mut raw = String::with_capacity(text.len());
44    for c in text.chars() {
45        match c {
46            '\n' => raw.push_str("\\n"),
47            '\t' => raw.push_str("\\t"),
48            '"' | '\\' | '{' | '}' => {
49                raw.push('\\');
50                raw.push(c);
51            }
52            _ => raw.push(c),
53        }
54    }
55    raw
56}
57
58/// The characters that may follow `\\` in a string.
59pub const ESCAPES: &[char] = &['n', 't', '"', '\\', '{', '}'];
60
61enum Mode {
62    /// `open_braces` counts `{` inside an interpolation, so its closing `}` can be found.
63    Code {
64        open_braces: u32,
65    },
66    String {
67        start: usize,
68    },
69}
70
71struct Lexer<'src> {
72    file: FileId,
73    source: &'src str,
74    bytes: &'src [u8],
75    pos: usize,
76    modes: Vec<Mode>,
77    tokens: Vec<Token>,
78    diagnostics: Vec<Diagnostic>,
79}
80
81impl Lexer<'_> {
82    fn run(&mut self) {
83        while self.pos < self.bytes.len() {
84            match self.modes.last() {
85                Some(Mode::String { start }) => self.lex_string_part(*start),
86                _ => self.lex_code(),
87            }
88        }
89        if let Some(Mode::String { start }) = self
90            .modes
91            .iter()
92            .rev()
93            .find(|mode| matches!(mode, Mode::String { .. }))
94        {
95            let span = self.span(*start, self.pos);
96            self.error("this string is never closed", span);
97        }
98        self.push(TokenKind::Eof, self.pos);
99    }
100
101    fn lex_code(&mut self) {
102        let start = self.pos;
103        let byte = self.bytes[start];
104        self.pos += 1;
105        let kind = match byte {
106            b' ' | b'\t' | b'\r' => return,
107            b'\n' => {
108                if self
109                    .tokens
110                    .last()
111                    .is_some_and(|t| t.kind != TokenKind::Newline)
112                {
113                    self.push(TokenKind::Newline, start);
114                }
115                return;
116            }
117            b'/' if self.peek() == Some(b'/') => {
118                while self.peek().is_some_and(|b| b != b'\n') {
119                    self.pos += 1;
120                }
121                return;
122            }
123            b'"' => {
124                self.push(TokenKind::StringStart, start);
125                self.modes.push(Mode::String { start });
126                return;
127            }
128            b'0'..=b'9' => self.number(),
129            b'a'..=b'z' | b'A'..=b'Z' | b'_' => self.identifier(start),
130            b'{' => {
131                if let Some(Mode::Code { open_braces }) = self.modes.last_mut() {
132                    *open_braces += 1;
133                }
134                TokenKind::LBrace
135            }
136            b'}' => self.close_brace(),
137            b'(' => TokenKind::LParen,
138            b')' => TokenKind::RParen,
139            b'[' => TokenKind::LBracket,
140            b']' => TokenKind::RBracket,
141            b',' => TokenKind::Comma,
142            b':' => TokenKind::Colon,
143            b'.' if self.eat(b'.') => TokenKind::DotDot,
144            b'.' => TokenKind::Dot,
145            b'$' => TokenKind::Dollar,
146            b'%' => TokenKind::Percent,
147            b'+' => self.with_eq(TokenKind::Plus, TokenKind::PlusEq),
148            b'*' => self.with_eq(TokenKind::Star, TokenKind::StarEq),
149            b'/' => self.with_eq(TokenKind::Slash, TokenKind::SlashEq),
150            b'=' => self.with_eq(TokenKind::Eq, TokenKind::EqEq),
151            b'!' => self.with_eq(TokenKind::Bang, TokenKind::BangEq),
152            b'<' => self.with_eq(TokenKind::Lt, TokenKind::LtEq),
153            b'>' => self.with_eq(TokenKind::Gt, TokenKind::GtEq),
154            b'-' if self.eat(b'>') => TokenKind::Arrow,
155            b'-' => self.with_eq(TokenKind::Minus, TokenKind::MinusEq),
156            b'&' | b'|' => {
157                if self.eat(byte) {
158                    if byte == b'&' {
159                        TokenKind::AndAnd
160                    } else {
161                        TokenKind::OrOr
162                    }
163                } else {
164                    let doubled = if byte == b'&' { "&&" } else { "||" };
165                    self.error(format!("expected `{doubled}`"), self.span(start, self.pos));
166                    return;
167                }
168            }
169            _ => {
170                self.unexpected_character(start);
171                return;
172            }
173        };
174        self.push(kind, start);
175    }
176
177    fn close_brace(&mut self) -> TokenKind {
178        let in_interpolation = self.modes.len() > 1;
179        match self.modes.last_mut() {
180            Some(Mode::Code { open_braces: 0 }) if in_interpolation => {
181                self.modes.pop();
182                TokenKind::InterpolationEnd
183            }
184            Some(Mode::Code { open_braces }) => {
185                *open_braces = open_braces.saturating_sub(1);
186                TokenKind::RBrace
187            }
188            _ => TokenKind::RBrace,
189        }
190    }
191
192    fn lex_string_part(&mut self, string_start: usize) {
193        let start = self.pos;
194        match self.bytes[start] {
195            b'"' => {
196                self.pos += 1;
197                self.modes.pop();
198                self.push(TokenKind::StringEnd, start);
199            }
200            b'{' => {
201                self.pos += 1;
202                self.push(TokenKind::InterpolationStart, start);
203                self.modes.push(Mode::Code { open_braces: 0 });
204            }
205            b'\n' => {
206                self.error(
207                    "this string is never closed",
208                    self.span(string_start, start),
209                );
210                self.modes.pop();
211                self.push(TokenKind::StringEnd, start);
212            }
213            _ => {
214                while let Some(byte) = self.peek() {
215                    match byte {
216                        b'"' | b'{' | b'\n' => break,
217                        b'\\' => self.escape(),
218                        _ => self.pos += 1,
219                    }
220                }
221                self.push(TokenKind::StringText, start);
222            }
223        }
224    }
225
226    fn escape(&mut self) {
227        let start = self.pos;
228        self.pos += 1;
229        let Some(escaped) = self.source[self.pos..].chars().next() else {
230            return;
231        };
232        if escaped == '\n' {
233            return;
234        }
235        self.pos += escaped.len_utf8();
236        if !ESCAPES.contains(&escaped) {
237            let valid: Vec<_> = ESCAPES.iter().map(|c| format!("\\{c}")).collect();
238            self.error(
239                format!("unknown escape `\\{escaped}`; use {}", valid.join(" ")),
240                self.span(start, self.pos),
241            );
242        }
243    }
244
245    fn number(&mut self) -> TokenKind {
246        self.digits();
247        let fraction_follows = self.peek() == Some(b'.')
248            && self.bytes.get(self.pos + 1).is_some_and(u8::is_ascii_digit);
249        if !fraction_follows {
250            return TokenKind::Int;
251        }
252        self.pos += 1;
253        self.digits();
254        TokenKind::Float
255    }
256
257    fn digits(&mut self) {
258        while self.peek().is_some_and(|b| b.is_ascii_digit() || b == b'_') {
259            self.pos += 1;
260        }
261    }
262
263    fn identifier(&mut self, start: usize) -> TokenKind {
264        while self
265            .peek()
266            .is_some_and(|b| b.is_ascii_alphanumeric() || b == b'_')
267        {
268            self.pos += 1;
269        }
270        TokenKind::keyword(&self.source[start..self.pos]).unwrap_or(TokenKind::Ident)
271    }
272
273    fn unexpected_character(&mut self, start: usize) {
274        let Some(c) = self.source[start..].chars().next() else {
275            return;
276        };
277        self.pos = start + c.len_utf8();
278        self.error(
279            format!("unexpected character `{c}`"),
280            self.span(start, self.pos),
281        );
282    }
283
284    fn with_eq(&mut self, single: TokenKind, with_eq: TokenKind) -> TokenKind {
285        if self.eat(b'=') { with_eq } else { single }
286    }
287
288    fn peek(&self) -> Option<u8> {
289        self.bytes.get(self.pos).copied()
290    }
291
292    fn eat(&mut self, expected: u8) -> bool {
293        let matched = self.peek() == Some(expected);
294        if matched {
295            self.pos += 1;
296        }
297        matched
298    }
299
300    fn push(&mut self, kind: TokenKind, start: usize) {
301        let span = self.span(start, self.pos);
302        self.tokens.push(Token { kind, span });
303    }
304
305    fn span(&self, start: usize, end: usize) -> Span {
306        Span::new(self.file, start, end)
307    }
308
309    fn error(&mut self, message: impl Into<String>, span: Span) {
310        self.diagnostics.push(Diagnostic::new(message, span));
311    }
312}
313
314#[cfg(test)]
315mod tests {
316    use super::*;
317    use crate::token::{KEYWORDS, SYMBOLS};
318    use TokenKind::*;
319
320    fn kinds(source: &str) -> Vec<TokenKind> {
321        let (tokens, diagnostics) = lex(source, FileId::default());
322        assert_eq!(diagnostics, []);
323        tokens.into_iter().map(|token| token.kind).collect()
324    }
325
326    #[test]
327    fn splits_interpolated_strings() {
328        assert_eq!(
329            kinds(r#""a {b + 1} c""#),
330            [
331                StringStart,
332                StringText,
333                InterpolationStart,
334                Ident,
335                Plus,
336                Int,
337                InterpolationEnd,
338                StringText,
339                StringEnd,
340                Eof
341            ]
342        );
343    }
344
345    #[test]
346    fn braces_inside_interpolation_do_not_end_it() {
347        assert_eq!(
348            kinds(r#""{f() { x }}""#),
349            [
350                StringStart,
351                InterpolationStart,
352                Ident,
353                LParen,
354                RParen,
355                LBrace,
356                Ident,
357                RBrace,
358                InterpolationEnd,
359                StringEnd,
360                Eof
361            ]
362        );
363    }
364
365    #[test]
366    fn collapses_blank_lines_and_skips_comments() {
367        assert_eq!(kinds("a // note\n\n\nb"), [Ident, Newline, Ident, Eof]);
368    }
369
370    #[test]
371    fn reads_numbers_operators_and_keywords() {
372        assert_eq!(
373            kinds("state x = 1_000 + 2.5 -> a.len() != -3"),
374            [
375                State, Ident, Eq, Int, Plus, Float, Arrow, Ident, Dot, Ident, LParen, RParen,
376                BangEq, Minus, Int, Eof
377            ]
378        );
379    }
380
381    #[test]
382    fn every_keyword_and_symbol_lexes_to_its_token() {
383        for &kind in KEYWORDS.iter().chain(SYMBOLS) {
384            let text = kind.text().expect("keywords and symbols have fixed text");
385            assert_eq!(kinds(text), [kind, Eof], "lexing `{text}`");
386        }
387    }
388
389    #[test]
390    fn unescapes_string_text() {
391        let source = r#""say \"hi\"\n\{x\}""#;
392        let (tokens, diagnostics) = lex(source, FileId::default());
393        assert_eq!(diagnostics, []);
394        assert_eq!(unescape(&source[tokens[1].span.range()]), "say \"hi\"\n{x}");
395    }
396
397    #[test]
398    fn escaped_text_lexes_cleanly_and_unescapes_back() {
399        let text = "C:\\Users\\me\t\"{x}\"\n";
400        let source = format!("\"{}\"", escape(text));
401        let (tokens, diagnostics) = lex(&source, FileId::default());
402        assert_eq!(diagnostics, []);
403        assert_eq!(unescape(&source[tokens[1].span.range()]), text);
404    }
405
406    #[test]
407    fn reports_unknown_escapes_and_unclosed_strings() {
408        let (_, diagnostics) = lex("\"a\\qb\nx", FileId::default());
409        let messages: Vec<_> = diagnostics.iter().map(|d| d.message.as_str()).collect();
410        assert_eq!(
411            messages,
412            [
413                "unknown escape `\\q`; use \\n \\t \\\" \\\\ \\{ \\}",
414                "this string is never closed"
415            ]
416        );
417    }
418}
nifty_syntax/lexer.rs

nifty_syntax/
lexer.rs
