ephemeris_documents/json/
lexer.rs

1//! Tokens of JSON5 source.
2//!
3//! Whitespace and comments are tokens too, so the parser can keep them.
4//!
5//! Identifier characters are approximated with `char::is_alphabetic` and
6//! `char::is_alphanumeric` plus `$`, `_`, U+200C and U+200D. That covers the
7//! ECMAScript letter categories and decimal digits, but misses some combining
8//! marks and connector punctuation. Whitespace is exact: the ECMAScript list
9//! and the `Zs` category.
10
11use crate::error::{JsonError, JsonErrorKind};
12
13#[derive(Debug, Clone, PartialEq, Eq)]
14pub(super) enum TokenKind {
15    OpenBrace,
16    CloseBrace,
17    OpenBracket,
18    CloseBracket,
19    Colon,
20    Comma,
21    /// A quoted string and its decoded text. `None` when the text holds a lone
22    /// surrogate, which a Rust string cannot.
23    String(Option<String>),
24    /// An identifier name and its decoded text.
25    Identifier(String),
26    Number,
27    Whitespace,
28    LineComment,
29    BlockComment,
30}
31
32impl TokenKind {
33    pub(super) const fn is_trivia(&self) -> bool {
34        matches!(
35            self,
36            Self::Whitespace | Self::LineComment | Self::BlockComment
37        )
38    }
39}
40
41#[derive(Debug, Clone, PartialEq, Eq)]
42pub(super) struct Token<'a> {
43    pub(super) kind: TokenKind,
44    pub(super) start: usize,
45    pub(super) text: &'a str,
46}
47
48pub(super) struct Lexer<'a> {
49    source: &'a str,
50    position: usize,
51}
52
53impl<'a> Lexer<'a> {
54    /// Split `source` into tokens that cover it completely.
55    ///
56    /// # Errors
57    /// Returns the first lexical error.
58    pub(super) fn tokenize(source: &'a str) -> Result<Vec<Token<'a>>, JsonError> {
59        let mut lexer = Self {
60            source,
61            position: 0,
62        };
63        let mut tokens = Vec::new();
64        while let Some(token) = lexer.token()? {
65            tokens.push(token);
66        }
67        Ok(tokens)
68    }
69
70    pub(super) const fn is_line_terminator(character: char) -> bool {
71        matches!(character, '\n' | '\r' | '\u{2028}' | '\u{2029}')
72    }
73
74    const fn is_whitespace(character: char) -> bool {
75        matches!(
76            character,
77            '\t' | '\u{b}' | '\u{c}' | ' ' | '\u{a0}' | '\u{1680}' | '\u{2000}'
78                ..='\u{200a}' | '\u{202f}' | '\u{205f}' | '\u{3000}' | '\u{feff}'
79        )
80    }
81
82    fn is_identifier_start(character: char) -> bool {
83        character.is_alphabetic() || character == '$' || character == '_'
84    }
85
86    fn is_identifier_part(character: char) -> bool {
87        character.is_alphanumeric() || matches!(character, '$' | '_' | '\u{200c}' | '\u{200d}')
88    }
89
90    const fn error(offset: usize, kind: JsonErrorKind) -> JsonError {
91        JsonError { offset, kind }
92    }
93
94    fn rest(&self) -> &'a str {
95        self.source.get(self.position..).unwrap_or_default()
96    }
97
98    fn peek(&self) -> Option<char> {
99        self.rest().chars().next()
100    }
101
102    fn peek_at(&self, index: usize) -> Option<char> {
103        self.rest().chars().nth(index)
104    }
105
106    fn bump(&mut self) -> Option<char> {
107        let character = self.peek()?;
108        self.position = self.position.saturating_add(character.len_utf8());
109        Some(character)
110    }
111
112    fn eat(&mut self, expected: char) -> bool {
113        let found = self.peek() == Some(expected);
114        if found {
115            self.bump();
116        }
117        found
118    }
119
120    /// Consume characters while `accept` holds and count them.
121    fn eat_while(&mut self, accept: impl Fn(char) -> bool) -> usize {
122        let mut count = 0usize;
123        while self.peek().is_some_and(&accept) {
124            self.bump();
125            count = count.saturating_add(1);
126        }
127        count
128    }
129
130    fn token(&mut self) -> Result<Option<Token<'a>>, JsonError> {
131        let start = self.position;
132        let Some(first) = self.peek() else {
133            return Ok(None);
134        };
135        let kind = match first {
136            '{' => self.punctuation(TokenKind::OpenBrace),
137            '}' => self.punctuation(TokenKind::CloseBrace),
138            '[' => self.punctuation(TokenKind::OpenBracket),
139            ']' => self.punctuation(TokenKind::CloseBracket),
140            ':' => self.punctuation(TokenKind::Colon),
141            ',' => self.punctuation(TokenKind::Comma),
142            '"' | '\'' => TokenKind::String(self.string(first)?),
143            '/' => self.comment()?,
144            '+' | '-' | '.' | '0'..='9' => self.number()?,
145            '\\' => TokenKind::Identifier(self.identifier()?),
146            character if Self::is_identifier_start(character) => {
147                TokenKind::Identifier(self.identifier()?)
148            }
149            character if Self::is_whitespace(character) || Self::is_line_terminator(character) => {
150                self.eat_while(|next| Self::is_whitespace(next) || Self::is_line_terminator(next));
151                TokenKind::Whitespace
152            }
153            found => {
154                return Err(Self::error(
155                    start,
156                    JsonErrorKind::UnexpectedCharacter { found },
157                ));
158            }
159        };
160        let text = self.source.get(start..self.position).unwrap_or_default();
161        Ok(Some(Token { kind, start, text }))
162    }
163
164    fn punctuation(&mut self, kind: TokenKind) -> TokenKind {
165        self.bump();
166        kind
167    }
168
169    fn comment(&mut self) -> Result<TokenKind, JsonError> {
170        let start = self.position;
171        match self.peek_at(1) {
172            Some('/') => {
173                self.eat_while(|next| !Self::is_line_terminator(next));
174                Ok(TokenKind::LineComment)
175            }
176            Some('*') => {
177                let body = self.rest().get(2..).unwrap_or_default();
178                let Some(end) = body.find("*/") else {
179                    return Err(Self::error(start, JsonErrorKind::UnterminatedComment));
180                };
181                self.position = self
182                    .position
183                    .saturating_add(2)
184                    .saturating_add(end)
185                    .saturating_add(2);
186                Ok(TokenKind::BlockComment)
187            }
188            _ => Err(Self::error(
189                start,
190                JsonErrorKind::UnexpectedCharacter { found: '/' },
191            )),
192        }
193    }
194
195    fn string(&mut self, quote: char) -> Result<Option<String>, JsonError> {
196        let start = self.position;
197        self.bump();
198        let mut decoded = Some(String::new());
199        loop {
200            match self.bump() {
201                None | Some('\n' | '\r') => {
202                    return Err(Self::error(start, JsonErrorKind::UnterminatedString));
203                }
204                Some(character) if character == quote => return Ok(decoded),
205                Some('\\') => self.escape(start, &mut decoded)?,
206                Some(character) => Self::push(&mut decoded, character),
207            }
208        }
209    }
210
211    fn push(decoded: &mut Option<String>, character: char) {
212        if let Some(text) = decoded {
213            text.push(character);
214        }
215    }
216
217    /// Read the rest of an escape in a string. The backslash is consumed.
218    fn escape(
219        &mut self,
220        string_start: usize,
221        decoded: &mut Option<String>,
222    ) -> Result<(), JsonError> {
223        let backslash = self.position.saturating_sub(1);
224        let invalid = || Self::error(backslash, JsonErrorKind::InvalidEscape);
225        let Some(character) = self.bump() else {
226            return Err(Self::error(string_start, JsonErrorKind::UnterminatedString));
227        };
228        let value = match character {
229            'b' => '\u{8}',
230            'f' => '\u{c}',
231            'n' => '\n',
232            'r' => '\r',
233            't' => '\t',
234            'v' => '\u{b}',
235            '0' if self.peek().is_some_and(|next| next.is_ascii_digit()) => {
236                return Err(invalid());
237            }
238            '0' => '\0',
239            '1'..='9' => return Err(invalid()),
240            'x' => {
241                let code = self.hex_digits(2).ok_or_else(invalid)?;
242                char::from_u32(code).ok_or_else(invalid)?
243            }
244            'u' => {
245                let code = self.hex_digits(4).ok_or_else(invalid)?;
246                let Some(value) = self
247                    .surrogate_pair(code)
248                    .map_or_else(|| char::from_u32(code), Some)
249                else {
250                    *decoded = None;
251                    return Ok(());
252                };
253                value
254            }
255            '\r' => {
256                self.eat('\n');
257                return Ok(());
258            }
259            '\n' | '\u{2028}' | '\u{2029}' => return Ok(()),
260            other => other,
261        };
262        Self::push(decoded, value);
263        Ok(())
264    }
265
266    /// Combine a high surrogate with a following `\uXXXX` low surrogate.
267    fn surrogate_pair(&mut self, high: u32) -> Option<char> {
268        if !(0xD800..=0xDBFF).contains(&high) {
269            return None;
270        }
271        let saved = self.position;
272        if self.eat('\\')
273            && self.eat('u')
274            && let Some(low) = self.hex_digits(4)
275            && (0xDC00..=0xDFFF).contains(&low)
276        {
277            let offset = high.saturating_sub(0xD800).checked_shl(10)?;
278            let code = offset
279                .saturating_add(low.saturating_sub(0xDC00))
280                .saturating_add(0x1_0000);
281            return char::from_u32(code);
282        }
283        self.position = saved;
284        None
285    }
286
287    fn hex_digits(&mut self, count: usize) -> Option<u32> {
288        let mut value = 0u32;
289        for _ in 0..count {
290            let digit = self.peek()?.to_digit(16)?;
291            self.bump();
292            value = value.checked_mul(16)?.checked_add(digit)?;
293        }
294        Some(value)
295    }
296
297    fn identifier(&mut self) -> Result<String, JsonError> {
298        let mut name = String::new();
299        loop {
300            let first = name.is_empty();
301            let accepts = |character: char| {
302                if first {
303                    Self::is_identifier_start(character)
304                } else {
305                    Self::is_identifier_part(character)
306                }
307            };
308            match self.peek() {
309                Some('\\') => {
310                    let backslash = self.position;
311                    self.bump();
312                    let invalid = || Self::error(backslash, JsonErrorKind::InvalidEscape);
313                    if !self.eat('u') {
314                        return Err(invalid());
315                    }
316                    let code = self.hex_digits(4).ok_or_else(invalid)?;
317                    let character = char::from_u32(code)
318                        .filter(|&character| accepts(character))
319                        .ok_or_else(invalid)?;
320                    name.push(character);
321                }
322                Some(character) if accepts(character) => {
323                    self.bump();
324                    name.push(character);
325                }
326                _ => return Ok(name),
327            }
328        }
329    }
330
331    fn number(&mut self) -> Result<TokenKind, JsonError> {
332        let start = self.position;
333        let invalid = || Self::error(start, JsonErrorKind::InvalidNumber);
334        if matches!(self.peek(), Some('+' | '-')) {
335            self.bump();
336        }
337        match self.peek() {
338            Some('I' | 'N') => {
339                let word_start = self.position;
340                self.eat_while(|next| next.is_ascii_alphabetic());
341                let word = self.source.get(word_start..self.position);
342                if !matches!(word, Some("Infinity" | "NaN")) {
343                    return Err(invalid());
344                }
345            }
346            Some('0') if matches!(self.peek_at(1), Some('x' | 'X')) => {
347                self.bump();
348                self.bump();
349                if self.eat_while(|next| next.is_ascii_hexdigit()) == 0 {
350                    return Err(invalid());
351                }
352            }
353            _ => {
354                if !self.decimal() {
355                    return Err(invalid());
356                }
357            }
358        }
359        if self
360            .peek()
361            .is_some_and(|next| Self::is_identifier_part(next) || next == '\\')
362        {
363            return Err(invalid());
364        }
365        Ok(TokenKind::Number)
366    }
367
368    /// Decimal digits with optional fraction and exponent. Returns `false` when
369    /// the literal is malformed.
370    fn decimal(&mut self) -> bool {
371        let leading_zero = self.peek() == Some('0');
372        let integer = self.eat_while(|next| next.is_ascii_digit());
373        if leading_zero && integer > 1 {
374            return false;
375        }
376        let fraction = if self.eat('.') {
377            self.eat_while(|next| next.is_ascii_digit())
378        } else {
379            0
380        };
381        if integer == 0 && fraction == 0 {
382            return false;
383        }
384        if matches!(self.peek(), Some('e' | 'E')) {
385            self.bump();
386            if matches!(self.peek(), Some('+' | '-')) {
387                self.bump();
388            }
389            if self.eat_while(|next| next.is_ascii_digit()) == 0 {
390                return false;
391            }
392        }
393        true
394    }
395}
ephemeris_documents/json/lexer.rs

ephemeris_documents/json/
lexer.rs
