caisin/typst
1
0
Fork 0
mirror of https://github.com/typst/typst synced 2025-05-20 03:55:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

Better distinguish function names in tokenizer from parser ✂

Browse Source
This commit is contained in:
Laurenz 2020-07-26 18:08:08 +02:00
parent e7ffdde43d
commit ea64ce9aeb
1 changed files with 29 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
src/syntax/tokens.rs
View File

@ -188,7 +188,7 @@ impl<'s> Tokens<'s> {
} }
/// The line-colunn position in the source at which the last token ends and /// The line-colunn position in the source at which the last token ends and
/// next token will start. This position is /// next token will start.
pub fn pos(&self) -> Position { pub fn pos(&self) -> Position {
self.position self.position
} }
@ -204,15 +204,15 @@ impl<'s> Iterator for Tokens<'s> {
let token = match first { let token = match first {
// Comments. // Comments.
'/' if self.peek() == Some('/') => self.parse_line_comment(), '/' if self.peek() == Some('/') => self.read_line_comment(),
'/' if self.peek() == Some('*') => self.parse_block_comment(), '/' if self.peek() == Some('*') => self.read_block_comment(),
'*' if self.peek() == Some('/') => { self.eat(); Invalid("*/") } '*' if self.peek() == Some('/') => { self.eat(); Invalid("*/") }
// Whitespace. // Whitespace.
c if c.is_whitespace() => self.parse_whitespace(start), c if c.is_whitespace() => self.read_whitespace(start),
// Functions. // Functions.
'[' => self.parse_function(start), '[' => self.read_function(start),
']' => Invalid("]"), ']' => Invalid("]"),
// Syntactic elements in function headers. // Syntactic elements in function headers.
@ -230,7 +230,7 @@ impl<'s> Iterator for Tokens<'s> {
'/' if self.mode == Header => Slash, '/' if self.mode == Header => Slash,
// String values. // String values.
'"' if self.mode == Header => self.parse_string(), '"' if self.mode == Header => self.read_string(),
// Star serves a double purpose as a style modifier // Star serves a double purpose as a style modifier
// and a expression operator in the header. // and a expression operator in the header.
@ -238,13 +238,13 @@ impl<'s> Iterator for Tokens<'s> {
// Style toggles. // Style toggles.
'_' if self.mode == Body => Underscore, '_' if self.mode == Body => Underscore,
'`' if self.mode == Body => self.parse_raw(), '`' if self.mode == Body => self.read_raw(),
// An escaped thing. // An escaped thing.
'\\' if self.mode == Body => self.parse_escaped(), '\\' if self.mode == Body => self.read_escaped(),
// A hex expression. // A hex expression.
'#' if self.mode == Header => self.parse_hex_value(), '#' if self.mode == Header => self.read_hex(),
// Expressions or just strings. // Expressions or just strings.
c => { c => {
@ -267,7 +267,7 @@ impl<'s> Iterator for Tokens<'s> {
}, false, -(c.len_utf8() as isize), 0).0; }, false, -(c.len_utf8() as isize), 0).0;
if self.mode == Header { if self.mode == Header {
self.parse_expr(text) self.read_expr(text)
} else { } else {
Text(text) Text(text)
} }
@ -282,11 +282,11 @@ impl<'s> Iterator for Tokens<'s> {
} }
impl<'s> Tokens<'s> { impl<'s> Tokens<'s> {
fn parse_line_comment(&mut self) -> Token<'s> { fn read_line_comment(&mut self) -> Token<'s> {
LineComment(self.read_string_until(is_newline_char, false, 1, 0).0) LineComment(self.read_string_until(is_newline_char, false, 1, 0).0)
} }
fn parse_block_comment(&mut self) -> Token<'s> { fn read_block_comment(&mut self) -> Token<'s> {
enum Last { Slash, Star, Other } enum Last { Slash, Star, Other }
self.eat(); self.eat();
@ -314,14 +314,14 @@ impl<'s> Tokens<'s> {
}, true, 0, -2).0) }, true, 0, -2).0)
} }
fn parse_whitespace(&mut self, start: Position) -> Token<'s> { fn read_whitespace(&mut self, start: Position) -> Token<'s> {
self.read_string_until(|n| !n.is_whitespace(), false, 0, 0); self.read_string_until(|n| !n.is_whitespace(), false, 0, 0);
let end = self.pos(); let end = self.pos();
Space(end.line - start.line) Space(end.line - start.line)
} }
fn parse_function(&mut self, start: Position) -> Token<'s> { fn read_function(&mut self, start: Position) -> Token<'s> {
let (header, terminated) = self.read_function_part(Header); let (header, terminated) = self.read_function_part(Header);
self.eat(); self.eat();
@ -353,11 +353,11 @@ impl<'s> Tokens<'s> {
self.eat(); self.eat();
match n { match n {
'[' => { self.parse_function(Position::ZERO); } '[' => { self.read_function(Position::ZERO); }
'/' if self.peek() == Some('/') => { self.parse_line_comment(); } '/' if self.peek() == Some('/') => { self.read_line_comment(); }
'/' if self.peek() == Some('*') => { self.parse_block_comment(); } '/' if self.peek() == Some('*') => { self.read_block_comment(); }
'"' if mode == Header => { self.parse_string(); } '"' if mode == Header => { self.read_string(); }
'`' if mode == Body => { self.parse_raw(); } '`' if mode == Body => { self.read_raw(); }
'\\' => { self.eat(); } '\\' => { self.eat(); }
_ => {} _ => {}
} }
@ -367,12 +367,12 @@ impl<'s> Tokens<'s> {
(&self.src[start .. end], terminated) (&self.src[start .. end], terminated)
} }
fn parse_string(&mut self) -> Token<'s> { fn read_string(&mut self) -> Token<'s> {
let (string, terminated) = self.read_until_unescaped('"'); let (string, terminated) = self.read_until_unescaped('"');
ExprStr { string, terminated } ExprStr { string, terminated }
} }
fn parse_raw(&mut self) -> Token<'s> { fn read_raw(&mut self) -> Token<'s> {
let (raw, terminated) = self.read_until_unescaped('`'); let (raw, terminated) = self.read_until_unescaped('`');
Raw { raw, terminated } Raw { raw, terminated }
} }
@ -390,7 +390,7 @@ impl<'s> Tokens<'s> {
}, true, 0, -1) }, true, 0, -1)
} }
fn parse_escaped(&mut self) -> Token<'s> { fn read_escaped(&mut self) -> Token<'s> {
fn is_escapable(c: char) -> bool { fn is_escapable(c: char) -> bool {
match c { match c {
'[' | ']' | '\\' | '/' | '*' | '_' | '`' | '"' => true, '[' | ']' | '\\' | '/' | '*' | '_' | '`' | '"' => true,
@ -410,7 +410,7 @@ impl<'s> Tokens<'s> {
} }
} }
fn parse_hex_value(&mut self) -> Token<'s> { fn read_hex(&mut self) -> Token<'s> {
// This will parse more than the permissable 0-9, a-f, A-F character // This will parse more than the permissable 0-9, a-f, A-F character
// ranges to provide nicer error messages later. // ranges to provide nicer error messages later.
ExprHex(self.read_string_until( ExprHex(self.read_string_until(
@ -419,7 +419,7 @@ impl<'s> Tokens<'s> {
).0) ).0)
} }
fn parse_expr(&mut self, text: &'s str) -> Token<'s> { fn read_expr(&mut self, text: &'s str) -> Token<'s> {
if let Ok(b) = text.parse::<bool>() { if let Ok(b) = text.parse::<bool>() {
ExprBool(b) ExprBool(b)
} else if let Ok(num) = text.parse::<f64>() { } else if let Ok(num) = text.parse::<f64>() {
@ -435,8 +435,11 @@ impl<'s> Tokens<'s> {
} }
} }
/// Will read the input stream until the argument F evaluates to `true` /// Will read the input stream until `f` evaluates to `true`. When
/// for the current character. /// `eat_match` is true, the token for which `f` was true is consumed.
/// Returns the string from the index where this was called offset by
/// `offset_start` to the end offset by `offset_end`. The end is before or
/// after the match depending on `eat_match`.
fn read_string_until<F>( fn read_string_until<F>(
&mut self, &mut self,
mut f: F, mut f: F,