From ea64ce9aeba411c7dc2f75f39a41935718d03122 Mon Sep 17 00:00:00 2001 From: Laurenz Date: Sun, 26 Jul 2020 18:08:08 +0200 Subject: [PATCH 1/4] =?UTF-8?q?Better=20distinguish=20function=20names=20i?= =?UTF-8?q?n=20tokenizer=20from=20parser=20=E2=9C=82?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/syntax/tokens.rs | 55 +++++++++++++++++++++++--------------------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs index 3b34019d9..0fc52f266 100644 --- a/src/syntax/tokens.rs +++ b/src/syntax/tokens.rs @@ -188,7 +188,7 @@ impl<'s> Tokens<'s> { } /// The line-colunn position in the source at which the last token ends and - /// next token will start. This position is + /// next token will start. pub fn pos(&self) -> Position { self.position } @@ -204,15 +204,15 @@ impl<'s> Iterator for Tokens<'s> { let token = match first { // Comments. - '/' if self.peek() == Some('/') => self.parse_line_comment(), - '/' if self.peek() == Some('*') => self.parse_block_comment(), + '/' if self.peek() == Some('/') => self.read_line_comment(), + '/' if self.peek() == Some('*') => self.read_block_comment(), '*' if self.peek() == Some('/') => { self.eat(); Invalid("*/") } // Whitespace. - c if c.is_whitespace() => self.parse_whitespace(start), + c if c.is_whitespace() => self.read_whitespace(start), // Functions. - '[' => self.parse_function(start), + '[' => self.read_function(start), ']' => Invalid("]"), // Syntactic elements in function headers. @@ -230,7 +230,7 @@ impl<'s> Iterator for Tokens<'s> { '/' if self.mode == Header => Slash, // String values. - '"' if self.mode == Header => self.parse_string(), + '"' if self.mode == Header => self.read_string(), // Star serves a double purpose as a style modifier // and a expression operator in the header. @@ -238,13 +238,13 @@ impl<'s> Iterator for Tokens<'s> { // Style toggles. '_' if self.mode == Body => Underscore, - '`' if self.mode == Body => self.parse_raw(), + '`' if self.mode == Body => self.read_raw(), // An escaped thing. - '\\' if self.mode == Body => self.parse_escaped(), + '\\' if self.mode == Body => self.read_escaped(), // A hex expression. - '#' if self.mode == Header => self.parse_hex_value(), + '#' if self.mode == Header => self.read_hex(), // Expressions or just strings. c => { @@ -267,7 +267,7 @@ impl<'s> Iterator for Tokens<'s> { }, false, -(c.len_utf8() as isize), 0).0; if self.mode == Header { - self.parse_expr(text) + self.read_expr(text) } else { Text(text) } @@ -282,11 +282,11 @@ impl<'s> Iterator for Tokens<'s> { } impl<'s> Tokens<'s> { - fn parse_line_comment(&mut self) -> Token<'s> { + fn read_line_comment(&mut self) -> Token<'s> { LineComment(self.read_string_until(is_newline_char, false, 1, 0).0) } - fn parse_block_comment(&mut self) -> Token<'s> { + fn read_block_comment(&mut self) -> Token<'s> { enum Last { Slash, Star, Other } self.eat(); @@ -314,14 +314,14 @@ impl<'s> Tokens<'s> { }, true, 0, -2).0) } - fn parse_whitespace(&mut self, start: Position) -> Token<'s> { + fn read_whitespace(&mut self, start: Position) -> Token<'s> { self.read_string_until(|n| !n.is_whitespace(), false, 0, 0); let end = self.pos(); Space(end.line - start.line) } - fn parse_function(&mut self, start: Position) -> Token<'s> { + fn read_function(&mut self, start: Position) -> Token<'s> { let (header, terminated) = self.read_function_part(Header); self.eat(); @@ -353,11 +353,11 @@ impl<'s> Tokens<'s> { self.eat(); match n { - '[' => { self.parse_function(Position::ZERO); } - '/' if self.peek() == Some('/') => { self.parse_line_comment(); } - '/' if self.peek() == Some('*') => { self.parse_block_comment(); } - '"' if mode == Header => { self.parse_string(); } - '`' if mode == Body => { self.parse_raw(); } + '[' => { self.read_function(Position::ZERO); } + '/' if self.peek() == Some('/') => { self.read_line_comment(); } + '/' if self.peek() == Some('*') => { self.read_block_comment(); } + '"' if mode == Header => { self.read_string(); } + '`' if mode == Body => { self.read_raw(); } '\\' => { self.eat(); } _ => {} } @@ -367,12 +367,12 @@ impl<'s> Tokens<'s> { (&self.src[start .. end], terminated) } - fn parse_string(&mut self) -> Token<'s> { + fn read_string(&mut self) -> Token<'s> { let (string, terminated) = self.read_until_unescaped('"'); ExprStr { string, terminated } } - fn parse_raw(&mut self) -> Token<'s> { + fn read_raw(&mut self) -> Token<'s> { let (raw, terminated) = self.read_until_unescaped('`'); Raw { raw, terminated } } @@ -390,7 +390,7 @@ impl<'s> Tokens<'s> { }, true, 0, -1) } - fn parse_escaped(&mut self) -> Token<'s> { + fn read_escaped(&mut self) -> Token<'s> { fn is_escapable(c: char) -> bool { match c { '[' | ']' | '\\' | '/' | '*' | '_' | '`' | '"' => true, @@ -410,7 +410,7 @@ impl<'s> Tokens<'s> { } } - fn parse_hex_value(&mut self) -> Token<'s> { + fn read_hex(&mut self) -> Token<'s> { // This will parse more than the permissable 0-9, a-f, A-F character // ranges to provide nicer error messages later. ExprHex(self.read_string_until( @@ -419,7 +419,7 @@ impl<'s> Tokens<'s> { ).0) } - fn parse_expr(&mut self, text: &'s str) -> Token<'s> { + fn read_expr(&mut self, text: &'s str) -> Token<'s> { if let Ok(b) = text.parse::() { ExprBool(b) } else if let Ok(num) = text.parse::() { @@ -435,8 +435,11 @@ impl<'s> Tokens<'s> { } } - /// Will read the input stream until the argument F evaluates to `true` - /// for the current character. + /// Will read the input stream until `f` evaluates to `true`. When + /// `eat_match` is true, the token for which `f` was true is consumed. + /// Returns the string from the index where this was called offset by + /// `offset_start` to the end offset by `offset_end`. The end is before or + /// after the match depending on `eat_match`. fn read_string_until( &mut self, mut f: F, From 9f400042cbb8aef7fa9b77b080f15a3701abf7a9 Mon Sep 17 00:00:00 2001 From: Laurenz Date: Sun, 26 Jul 2020 19:14:23 +0200 Subject: [PATCH 2/4] =?UTF-8?q?Streamline=20parser=20and=20tokenizer=20tes?= =?UTF-8?q?t=20code=20=E2=9C=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/syntax/expr.rs | 6 +- src/syntax/parsing.rs | 411 +++++++++++++++++++----------------------- src/syntax/span.rs | 207 +++++++++++---------- src/syntax/test.rs | 64 +++---- src/syntax/tokens.rs | 55 +++--- 5 files changed, 346 insertions(+), 397 deletions(-) diff --git a/src/syntax/expr.rs b/src/syntax/expr.rs index a27bdb62f..a1b3fd627 100644 --- a/src/syntax/expr.rs +++ b/src/syntax/expr.rs @@ -93,9 +93,6 @@ impl Debug for Expr { /// A unicode identifier. /// -/// The identifier must be valid! This is checked in [`Ident::new`] or -/// [`is_identifier`]. -/// /// # Example /// ```typst /// [func: "hi", ident] @@ -105,7 +102,8 @@ impl Debug for Expr { pub struct Ident(pub String); impl Ident { - /// Create a new identifier from a string checking that it is valid. + /// Create a new identifier from a string checking that it is a valid + /// unicode identifier. pub fn new(ident: S) -> Option where S: AsRef + Into { if is_identifier(ident.as_ref()) { Some(Ident(ident.into())) diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs index 09405d7fd..89d4f0fd2 100644 --- a/src/syntax/parsing.rs +++ b/src/syntax/parsing.rs @@ -639,84 +639,17 @@ fn unescape_raw(raw: &str) -> Vec { #[allow(non_snake_case)] mod tests { use crate::size::Size; - use crate::syntax::test::{DebugFn, check, zspan}; + use crate::syntax::test::{DebugFn, check}; use crate::syntax::func::Value; use super::*; use Decoration::*; + use Expr::{Number as Num, Size as Sz, Bool}; use Node::{ Space as S, ToggleItalic as Italic, ToggleBolder as Bold, Parbreak, Linebreak, }; - use Expr::{Number as Num, Size as Sz, Bool}; - fn Id(text: &str) -> Expr { Expr::Ident(Ident(text.to_string())) } - fn Str(text: &str) -> Expr { Expr::Str(text.to_string()) } - fn Pt(points: f32) -> Expr { Expr::Size(Size::pt(points)) } - fn Neg(e1: Expr) -> Expr { Expr::Neg(Box::new(zspan(e1))) } - fn Add(e1: Expr, e2: Expr) -> Expr { - Expr::Add(Box::new(zspan(e1)), Box::new(zspan(e2))) - } - fn Sub(e1: Expr, e2: Expr) -> Expr { - Expr::Sub(Box::new(zspan(e1)), Box::new(zspan(e2))) - } - fn Mul(e1: Expr, e2: Expr) -> Expr { - Expr::Mul(Box::new(zspan(e1)), Box::new(zspan(e2))) - } - fn Div(e1: Expr, e2: Expr) -> Expr { - Expr::Div(Box::new(zspan(e1)), Box::new(zspan(e2))) - } - - fn Clr(r: u8, g: u8, b: u8, a: u8) -> Expr { - Expr::Color(RgbaColor::new(r, g, b, a)) - } - fn ClrStr(color: &str) -> Expr { - Expr::Color(RgbaColor::from_str(color).expect("invalid test color")) - } - fn ClrStrHealed() -> Expr { - let mut c = RgbaColor::from_str("000f").expect("invalid test color"); - c.healed = true; - Expr::Color(c) - } - - fn T(text: &str) -> Node { Node::Text(text.to_string()) } - - /// Create a raw text node. - macro_rules! raw { - ($($line:expr),* $(,)?) => { - Node::Raw(vec![$($line.to_string()),*]) - }; - } - - /// Create a tuple expression. - macro_rules! tuple { - ($($items:expr),* $(,)?) => { - Expr::Tuple(Tuple { items: spanned![vec $($items),*].0 }) - }; - } - - /// Create a named tuple expression. - macro_rules! named_tuple { - ($name:expr $(, $items:expr)* $(,)?) => { - Expr::NamedTuple(NamedTuple::new( - zspan(Ident($name.to_string())), - zspan(Tuple { items: spanned![vec $($items),*].0 }) - )) - }; - } - - /// Create an object expression. - macro_rules! object { - ($($key:expr => $value:expr),* $(,)?) => { - Expr::Object(Object { - pairs: vec![$(zspan(Pair { - key: zspan(Ident($key.to_string())), - value: zspan($value), - })),*] - }) - }; - } - /// Test whether the given string parses into /// - the given node list (required). /// - the given error list (optional, if omitted checks against empty list). @@ -736,12 +669,12 @@ mod tests { let ctx = ParseContext { scope: &scope }; let pass = parse(Position::ZERO, $source, ctx); - // Test model - let (exp, cmp) = spanned![vec $($model)*]; + // Test model. + let (exp, cmp) = span_vec![$($model)*]; check($source, exp, pass.output.nodes, cmp); - // Test problems - let (exp, cmp) = spanned![vec $($problems)*]; + // Test problems. + let (exp, cmp) = span_vec![$($problems)*]; let exp = exp.into_iter() .map(|s: Spanned<&str>| s.map(|e| e.to_string())) .collect::>(); @@ -750,42 +683,90 @@ mod tests { .collect::>(); check($source, exp, found, cmp); - // Test decos - $(let (exp, cmp) = spanned![vec $($decos)*]; + // Test decos. + $(let (exp, cmp) = span_vec![$($decos)*]; check($source, exp, pass.feedback.decos, cmp);)? }; } - /// Write down a `DebugFn` function model compactly. + /// Shorthand for `p!("[val: ...]" => func!("val", ...))`. + macro_rules! pval { + ($header:expr => $($tts:tt)*) => { + p!(concat!("[val: ", $header, "]") => [func!("val": $($tts)*)]); + } + } + + fn Id(text: &str) -> Expr { Expr::Ident(Ident(text.to_string())) } + fn Str(text: &str) -> Expr { Expr::Str(text.to_string()) } + fn Pt(points: f32) -> Expr { Expr::Size(Size::pt(points)) } + fn Color(r: u8, g: u8, b: u8, a: u8) -> Expr { Expr::Color(RgbaColor::new(r, g, b, a)) } + fn ColorStr(color: &str) -> Expr { Expr::Color(RgbaColor::from_str(color).expect("invalid test color")) } + fn ColorHealed() -> Expr { Expr::Color(RgbaColor::new_healed(0, 0, 0, 255)) } + fn Neg(e1: Expr) -> Expr { Expr::Neg(Box::new(Z(e1))) } + fn Add(e1: Expr, e2: Expr) -> Expr { Expr::Add(Box::new(Z(e1)), Box::new(Z(e2))) } + fn Sub(e1: Expr, e2: Expr) -> Expr { Expr::Sub(Box::new(Z(e1)), Box::new(Z(e2))) } + fn Mul(e1: Expr, e2: Expr) -> Expr { Expr::Mul(Box::new(Z(e1)), Box::new(Z(e2))) } + fn Div(e1: Expr, e2: Expr) -> Expr { Expr::Div(Box::new(Z(e1)), Box::new(Z(e2))) } + fn T(text: &str) -> Node { Node::Text(text.to_string()) } + fn Z(v: T) -> Spanned { Spanned::zero(v) } + + macro_rules! tuple { + ($($items:expr),* $(,)?) => { + Expr::Tuple(Tuple { items: span_vec![$($items),*].0 }) + }; + } + + macro_rules! named_tuple { + ($name:expr $(, $items:expr)* $(,)?) => { + Expr::NamedTuple(NamedTuple::new( + Z(Ident($name.to_string())), + Z(Tuple { items: span_vec![$($items),*].0 }) + )) + }; + } + + macro_rules! object { + ($($key:expr => $value:expr),* $(,)?) => { + Expr::Object(Object { + pairs: vec![$(Z(Pair { + key: Z(Ident($key.to_string())), + value: Z($value), + })),*] + }) + }; + } + + macro_rules! raw { + ($($line:expr),* $(,)?) => { + Node::Raw(vec![$($line.to_string()),*]) + }; + } + macro_rules! func { - ($name:tt $(: ($($pos:tt)*), { $($key:tt)* } )? $(; $($body:tt)*)?) => ({ + ($name:tt $(: ($($pos:tt)*) $(, { $($key:tt)* })? )? $(; $($body:tt)*)?) => {{ #[allow(unused_mut)] let mut args = FuncArgs::new(); - $(args.pos = Tuple::parse(zspan(tuple!($($pos)*))).unwrap();)? - $(args.key = Object::parse(zspan(object! { $($key)* })).unwrap();)? - + $(args.pos = Tuple::parse(Z(tuple!($($pos)*))).unwrap(); + $(args.key = Object::parse(Z(object! { $($key)* })).unwrap();)?)? Node::Model(Box::new(DebugFn { header: FuncHeader { - name: spanned!(item $name).map(|s| Ident(s.to_string())), + name: span_item!($name).map(|s| Ident(s.to_string())), args, }, body: func!(@body $($($body)*)?), })) - }); - - (@body [$($body:tt)*]) => ({ - Some(SyntaxModel { nodes: spanned![vec $($body)*].0 }) - }); - (@body) => (None); + }}; + (@body [$($body:tt)*]) => { Some(SyntaxModel { nodes: span_vec![$($body)*].0 }) }; + (@body) => { None }; } #[test] fn parse_color_strings() { - assert_eq!(Clr(0xf6, 0x12, 0x43, 0xff), ClrStr("f61243ff")); - assert_eq!(Clr(0xb3, 0xd8, 0xb3, 0xff), ClrStr("b3d8b3")); - assert_eq!(Clr(0xfc, 0xd2, 0xa9, 0xad), ClrStr("fCd2a9AD")); - assert_eq!(Clr(0x22, 0x33, 0x33, 0xff), ClrStr("233")); - assert_eq!(Clr(0x11, 0x11, 0x11, 0xbb), ClrStr("111b")); + assert_eq!(Color(0xf6, 0x12, 0x43, 0xff), ColorStr("f61243ff")); + assert_eq!(Color(0xb3, 0xd8, 0xb3, 0xff), ColorStr("b3d8b3")); + assert_eq!(Color(0xfc, 0xd2, 0xa9, 0xad), ColorStr("fCd2a9AD")); + assert_eq!(Color(0x22, 0x33, 0x33, 0xff), ColorStr("233")); + assert_eq!(Color(0x11, 0x11, 0x11, 0xbb), ColorStr("111b")); } #[test] @@ -824,7 +805,7 @@ mod tests { #[test] fn parse_basic_nodes() { - // Basic nodes + // Basic nodes. p!("" => []); p!("hi" => [T("hi")]); p!("*hi" => [Bold, T("hi")]); @@ -838,13 +819,13 @@ mod tests { p!(r"a\ b" => [T("a"), Linebreak, S, T("b")]); p!("πŸ’œ\n\n 🌍" => [T("πŸ’œ"), Parbreak, T("🌍")]); - // Raw markup + // Raw markup. p!("`py`" => [raw!["py"]]); p!("[val][`hi]`]" => [func!("val"; [raw!["hi]"]])]); p!("`hi\nyou" => [raw!["hi", "you"]], [(1:3, 1:3, "expected backtick")]); p!("`hi\\`du`" => [raw!["hi`du"]]); - // Spanned nodes + // Spanned nodes. p!("Hi" => [(0:0, 0:2, T("Hi"))]); p!("*Hi*" => [(0:0, 0:1, Bold), (0:1, 0:3, T("Hi")), (0:3, 0:4, Bold)]); p!("🌎\n*/[n]" => @@ -856,31 +837,31 @@ mod tests { #[test] fn parse_function_names() { - // No closing bracket + // No closing bracket. p!("[" => [func!("")], [ (0:1, 0:1, "expected identifier"), (0:1, 0:1, "expected closing bracket") ]); - // No name + // No name. p!("[]" => [func!("")], [(0:1, 0:1, "expected identifier")]); p!("[\"]" => [func!("")], [ (0:1, 0:3, "expected identifier, found string"), (0:3, 0:3, "expected closing bracket"), ]); - // An unknown name + // An unknown name. p!("[hi]" => [func!("hi")], [(0:1, 0:3, "unknown function")], [(0:1, 0:3, InvalidFuncName)], ); - // A valid name + // A valid name. p!("[f]" => [func!("f")], [], [(0:1, 0:2, ValidFuncName)]); p!("[ f]" => [func!("f")], [], [(0:3, 0:4, ValidFuncName)]); - // An invalid token for a name + // An invalid token for a name. p!("[12]" => [func!("")], [(0:1, 0:3, "expected identifier, found number")], []); p!("[🌎]" => [func!("")], [(0:1, 0:2, "expected identifier, found invalid token")], []); p!("[ 🌎]" => [func!("")], [(0:3, 0:4, "expected identifier, found invalid token")], []); @@ -888,13 +869,19 @@ mod tests { #[test] fn parse_colon_starting_function_arguments() { - // No colon before arg + // Valid. + p!("[val: true]" => + [func!["val": (Bool(true))]], [], + [(0:1, 0:4, ValidFuncName)], + ); + + // No colon before arg. p!("[val\"s\"]" => [func!("val")], [(0:4, 0:4, "expected colon")]); - // No colon before valid, but wrong token + // No colon before valid, but wrong token. p!("[val=]" => [func!("val")], [(0:4, 0:4, "expected colon")]); - // No colon before invalid tokens, which are ignored + // No colon before invalid tokens, which are ignored. p!("[val/🌎:$]" => [func!("val")], [(0:4, 0:4, "expected colon")], @@ -909,36 +896,38 @@ mod tests { (0:7, 0:7, "expected closing bracket"), ]); - // Just colon without args + // Just colon without args. p!("[val:]" => [func!("val")]); p!("[val:/*12pt*/]" => [func!("val")]); - // Whitespace / comments around colon - p!("[val\n:\ntrue]" => [func!("val": (Bool(true)), {})]); - p!("[val/*:*/://\ntrue]" => [func!("val": (Bool(true)), {})]); + // Whitespace / comments around colon. + p!("[val\n:\ntrue]" => [func!("val": (Bool(true)))]); + p!("[val/*:*/://\ntrue]" => [func!("val": (Bool(true)))]); } #[test] fn parse_one_positional_argument() { - // Different expressions - p!("[val: true]" => - [func!("val": (Bool(true)), {})], [], - [(0:1, 0:4, ValidFuncName)], - ); - p!("[val: _]" => [func!("val": (Id("_")), {})]); - p!("[val: name]" => [func!("val": (Id("name")), {})]); - p!("[val: \"hi\"]" => [func!("val": (Str("hi")), {})]); - p!("[val: \"a\n[]\\\"string\"]" => [func!("val": (Str("a\n[]\"string")), {})]); - p!("[val: 3.14]" => [func!("val": (Num(3.14)), {})]); - p!("[val: 4.5cm]" => [func!("val": (Sz(Size::cm(4.5))), {})]); - p!("[val: 12e1pt]" => [func!("val": (Pt(12e1)), {})]); - p!("[val: #f7a20500]" => [func!("val": (ClrStr("f7a20500")), {})]); + // Different expressions. + pval!("_" => (Id("_"))); + pval!("name" => (Id("name"))); + pval!("\"hi\"" => (Str("hi"))); + pval!("3.14" => (Num(3.14))); + pval!("4.5cm" => (Sz(Size::cm(4.5)))); + pval!("12e1pt" => (Pt(12e1))); + pval!("#f7a20500" => (ColorStr("f7a20500"))); + pval!("\"a\n[]\\\"string\"" => (Str("a\n[]\"string"))); - // Math - p!("[val: 3.2in + 6pt]" => [func!("val": (Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0)))), {})]); - p!("[val: 5 - 0.01]" => [func!("val": (Sub(Num(5.0), Num(0.01))), {})]); - p!("[val: (3mm * 2)]" => [func!("val": (Mul(Sz(Size::mm(3.0)), Num(2.0))), {})]); - p!("[val: 12e-3cm/1pt]" => [func!("val": (Div(Sz(Size::cm(12e-3)), Sz(Size::pt(1.0)))), {})]); + // Trailing comma. + pval!("a," => (Id("a"))); + + // Simple coerced tuple. + pval!("(hi)" => (Id("hi"))); + + // Math. + pval!("3.2in + 6pt" => (Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0))))); + pval!("5 - 0.01" => (Sub(Num(5.0), Num(0.01)))); + pval!("(3mm * 2)" => (Mul(Sz(Size::mm(3.0)), Num(2.0)))); + pval!("12e-3cm/1pt" => (Div(Sz(Size::cm(12e-3)), Sz(Size::pt(1.0))))); // Unclosed string. p!("[val: \"hello]" => [func!("val": (Str("hello]")), {})], [ @@ -946,112 +935,80 @@ mod tests { (0:13, 0:13, "expected closing bracket"), ]); - //Invalid colors - p!("[val: #12345]" => [func!("val": (ClrStrHealed()), {})], [ - (0:6, 0:12, "invalid color"), - ]); - p!("[val: #a5]" => [func!("val": (ClrStrHealed()), {})], [ - (0:6, 0:9, "invalid color"), - ]); - p!("[val: #14b2ah]" => [func!("val": (ClrStrHealed()), {})], [ - (0:6, 0:13, "invalid color"), - ]); - p!("[val: #f075ff011]" => [func!("val": (ClrStrHealed()), {})], [ - (0:6, 0:16, "invalid color"), - ]); + // Invalid, healed colors. + p!("[val: #12345]" => [func!("val": (ColorHealed()))], [(0:6, 0:12, "invalid color")]); + p!("[val: #a5]" => [func!("val": (ColorHealed()))], [(0:6, 0:9, "invalid color")]); + p!("[val: #14b2ah]" => [func!("val": (ColorHealed()))], [(0:6, 0:13, "invalid color")]); + p!("[val: #f075ff011]" => [func!("val": (ColorHealed()))], [(0:6, 0:16, "invalid color")]); } #[test] fn parse_complex_mathematical_expressions() { - p!("[val: (3.2in + 6pt)*(5/2-1)]" => [func!("val": ( - Mul( - Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0))), - Sub(Div(Num(5.0), Num(2.0)), Num(1.0)) - ) - ), {})]); - p!("[val: (6.3E+2+4* - 3.2pt)/2]" => [func!("val": ( - Div(Add(Num(6.3e2),Mul(Num(4.0), Neg(Pt(3.2)))), Num(2.0)) - ), {})]); - p!("[val: 4pt--]" => - [func!("val": (Pt(4.0)), {})], - [ - (0:10, 0:11, "dangling minus"), - (0:6, 0:10, "missing right summand") - ], - ); + // Valid expressions. + pval!("(3.2in + 6pt)*(5/2-1)" => (Mul( + Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0))), + Sub(Div(Num(5.0), Num(2.0)), Num(1.0)) + ))); + pval!("(6.3E+2+4* - 3.2pt)/2" => (Div( + Add(Num(6.3e2),Mul(Num(4.0), Neg(Pt(3.2)))), + Num(2.0) + ))); + + // Invalid expressions. + p!("[val: 4pt--]" => [func!("val": (Pt(4.0)))], [ + (0:10, 0:11, "dangling minus"), + (0:6, 0:10, "missing right summand") + ]); p!("[val: 3mm+4pt*]" => - [func!("val": (Add(Sz(Size::mm(3.0)), Pt(4.0))), {})], + [func!("val": (Add(Sz(Size::mm(3.0)), Pt(4.0))))], [(0:10, 0:14, "missing right factor")], ); } #[test] fn parse_tuples() { - // Empty tuple - p!("[val: ()]" => [func!("val": (tuple!()), {})]); - p!("[val: empty()]" => [func!("val": (named_tuple!("empty")), {})]); + // Empty tuple. + pval!("()" => (tuple!())); + pval!("empty()" => (named_tuple!("empty"))); - // Invalid value - p!("[val: (🌎)]" => - [func!("val": (tuple!()), {})], - [(0:7, 0:8, "expected value, found invalid token")], - ); + // Invalid value. p!("[val: sound(\x07)]" => [func!("val": (named_tuple!("sound")), {})], [(0:12, 0:13, "expected value, found invalid token")], ); - // Invalid tuple name + // Invalid tuple name. p!("[val: πŸ‘ (\"abc\", 13e-5)]" => [func!("val": (tuple!(Str("abc"), Num(13.0e-5))), {})], [(0:6, 0:7, "expected argument, found invalid token")], ); - // Unclosed tuple - p!("[val: (hello,]" => - [func!("val": (tuple!(Id("hello"),)), {})], - [(0:13, 0:13, "expected closing paren")], - ); + // Unclosed tuple. p!("[val: lang(δΈ­ζ–‡]" => [func!("val": (named_tuple!("lang", Id("δΈ­ζ–‡"))), {})], [(0:13, 0:13, "expected closing paren")], ); - // Valid values - p!("[val: (1, 2)]" => [func!("val": (tuple!(Num(1.0), Num(2.0))), {})]); - p!("[val: (\"s\",)]" => [func!("val": (tuple!(Str("s"))), {})]); - p!("[val: cmyk(1, 46, 0, 0)]" => - [func!("val": (named_tuple!( - "cmyk", Num(1.0), Num(46.0), Num(0.0), Num(0.0) - )), {})] - ); - p!("[val: items(\"fire\", #f93a6d)]" => - [func!("val": (named_tuple!( - "items", Str("fire"), ClrStr("f93a6d") - )), {})] - ); + // Valid values. + pval!("(1, 2)" => (tuple!(Num(1.0), Num(2.0)))); + pval!("(\"s\",)" => (tuple!(Str("s")))); + pval!("items(\"fire\", #f93a6d)" => ( + named_tuple!("items", Str("fire"), ColorStr("f93a6d") + ))); - // Nested tuples - p!("[val: (1, (2, 3))]" => - [func!("val": (tuple!(Num(1.0), tuple!(Num(2.0), Num(3.0)))), {})] - ); - p!("[val: css(1pt, rgb(90, 102, 254), \"solid\")]" => - [func!("val": (named_tuple!( - "css", Pt(1.0), named_tuple!( - "rgb", Num(90.0), Num(102.0), Num(254.0) - ), Str("solid") - )), {})] - ); + // Nested tuples. + pval!("css(1pt, rgb(90, 102, 254), \"solid\")" => (named_tuple!( + "css", + Pt(1.0), + named_tuple!("rgb", Num(90.0), Num(102.0), Num(254.0)), + Str("solid"), + ))); - // Invalid commas + // Invalid commas. p!("[val: (,)]" => [func!("val": (tuple!()), {})], [(0:7, 0:8, "expected value, found comma")], ); - p!("[val: nose(,)]" => - [func!("val": (named_tuple!("nose")), {})], - [(0:11, 0:12, "expected value, found comma")], - ); p!("[val: (true false)]" => [func!("val": (tuple!(Bool(true), Bool(false))), {})], [(0:11, 0:11, "expected comma")], @@ -1060,14 +1017,13 @@ mod tests { #[test] fn parse_objects() { - let f = || func!("val": (object! {}), {}); + let val = || func!("val": (object! {}), {}); - // Okay objects - p!("[val: {}]" => [f()]); - p!("[val: { key: value }]" => - [func!("val": (object! { "key" => Id("value") }), {})]); + // Okay objects. + pval!("{}" => (object! {})); + pval!("{ key: value }" => (object! { "key" => Id("value") })); - // Unclosed object + // Unclosed object. p!("[val: {hello: world]" => [func!("val": (object! { "hello" => Id("world") }), {})], [(0:19, 0:19, "expected closing brace")], @@ -1077,14 +1033,14 @@ mod tests { [(0:9, 0:9, "expected colon"), (0:9, 0:9, "expected closing brace")], ); - // Missing key - p!("[val: {,}]" => [f()], [(0:7, 0:8, "expected key, found comma")]); - p!("[val: { 12pt }]" => [f()], [(0:8, 0:12, "expected key, found size")]); - p!("[val: { : }]" => [f()], [(0:8, 0:9, "expected key, found colon")]); + // Missing key. + p!("[val: {,}]" => [val()], [(0:7, 0:8, "expected key, found comma")]); + p!("[val: { 12pt }]" => [val()], [(0:8, 0:12, "expected key, found size")]); + p!("[val: { : }]" => [val()], [(0:8, 0:9, "expected key, found colon")]); - // Missing colon - p!("[val: { key }]" => [f()], [(0:11, 0:11, "expected colon")]); - p!("[val: { key false }]" => [f()], [ + // Missing colon. + p!("[val: { key }]" => [val()], [(0:11, 0:11, "expected colon")]); + p!("[val: { key false }]" => [val()], [ (0:11, 0:11, "expected colon"), (0:12, 0:17, "expected key, found bool"), ]); @@ -1093,14 +1049,14 @@ mod tests { [(0:9, 0:9, "expected colon")], ); - // Missing value - p!("[val: { key: : }]" => [f()], [(0:13, 0:14, "expected value, found colon")]); + // Missing value. + p!("[val: { key: : }]" => [val()], [(0:13, 0:14, "expected value, found colon")]); p!("[val: { key: , k: \"s\" }]" => [func!("val": (object! { "k" => Str("s") }), {})], [(0:13, 0:14, "expected value, found comma")], ); - // Missing comma, invalid token + // Missing comma, invalid token. p!("[val: left={ a: 2, b: false 🌎 }]" => [func!("val": (), { "left" => object! { @@ -1115,7 +1071,7 @@ mod tests { #[test] fn parse_nested_tuples_and_objects() { - p!("[val: (1, { ab: (), d: (3, 14pt) }), false]" => [func!("val": ( + pval!("(1, { ab: (), d: (3, 14pt) }), false" => ( tuple!( Num(1.0), object!( @@ -1124,7 +1080,7 @@ mod tests { ), ), Bool(false), - ), {})]); + )); } #[test] @@ -1151,12 +1107,11 @@ mod tests { #[test] fn parse_multiple_mixed_arguments() { - p!("[val: a,]" => [func!("val": (Id("a")), {})]); p!("[val: 12pt, key=value]" => [func!("val": (Pt(12.0)), { "key" => Id("value") })], [], [(0:12, 0:15, ArgumentKey), (0:1, 0:4, ValidFuncName)], ); - p!("[val: a , \"b\" , c]" => [func!("val": (Id("a"), Str("b"), Id("c")), {})]); + pval!("a , x=\"b\" , c" => (Id("a"), Id("c")), { "x" => Str("b"), }); } #[test] @@ -1177,7 +1132,7 @@ mod tests { #[test] fn parse_invalid_key_value_pairs() { - // Invalid keys + // Invalid keys. p!("[val: true=you]" => [func!("val": (Bool(true), Id("you")), {})], [(0:10, 0:10, "expected comma"), @@ -1185,13 +1140,14 @@ mod tests { [(0:1, 0:4, ValidFuncName)], ); + // Unexpected equals. p!("[box: z=y=4]" => [func!("box": (Num(4.0)), { "z" => Id("y") })], [(0:9, 0:9, "expected comma"), (0:9, 0:10, "expected argument, found equals sign")], ); - // Invalid colon after keyable positional argument + // Invalid colon after keyable positional argument. p!("[val: key:12]" => [func!("val": (Id("key"), Num(12.0)), {})], [(0:9, 0:9, "expected comma"), @@ -1199,7 +1155,7 @@ mod tests { [(0:1, 0:4, ValidFuncName)], ); - // Invalid colon after non-keyable positional argument + // Invalid colon after unkeyable positional argument. p!("[val: true:12]" => [func!("val": (Bool(true), Num(12.0)), {})], [(0:10, 0:10, "expected comma"), (0:10, 0:11, "expected argument, found colon")], @@ -1209,7 +1165,7 @@ mod tests { #[test] fn parse_invalid_commas() { - // Missing commas + // Missing commas. p!("[val: 1pt 1]" => [func!("val": (Pt(1.0), Num(1.0)), {})], [(0:9, 0:9, "expected comma")], @@ -1219,7 +1175,7 @@ mod tests { [(0:7, 0:7, "expected comma")], ); - // Unexpected commas + // Unexpected commas. p!("[val:,]" => [func!("val")], [(0:5, 0:6, "expected argument, found comma")]); p!("[val: key=,]" => [func!("val")], [(0:10, 0:11, "expected value, found comma")]); p!("[val:, true]" => @@ -1231,13 +1187,10 @@ mod tests { #[test] fn parse_bodies() { p!("[val][Hi]" => [func!("val"; [T("Hi")])]); - - // Body nodes in bodies. p!("[val:*][*Hi*]" => [func!("val"; [Bold, T("Hi"), Bold])], [(0:5, 0:6, "expected argument, found star")], ); - // Errors in bodies. p!(" [val][ */ ]" => [S, func!("val"; [S, S])], @@ -1267,7 +1220,8 @@ mod tests { (0:6, 0:18, func!((0:1, 0:4, "val"); [(0:6, 0:11, T("world"))])), (0:18, 0:19, S), (0:19, 0:20, T("🌎")) - ], [], + ], + [], [(0:7, 0:10, ValidFuncName)], ); @@ -1281,7 +1235,8 @@ mod tests { (1:4, 1:10, func!((0:2, 0:5, "box"))), (1:10, 2:1, S), ])) - ], [], + ], + [], [(0:2, 0:5, ValidFuncName), (1:6, 1:9, ValidFuncName)], ); } diff --git a/src/syntax/span.rs b/src/syntax/span.rs index 9eb80d921..8083bc064 100644 --- a/src/syntax/span.rs +++ b/src/syntax/span.rs @@ -4,6 +4,114 @@ use std::fmt::{self, Debug, Formatter}; use std::ops::{Add, Sub}; use serde::Serialize; +/// A vector of spanned things. +pub type SpanVec = Vec>; + +/// [Offset](Span::offset) all spans in a vector of spanned things by a start +/// position. +pub fn offset_spans( + start: Position, + vec: SpanVec, +) -> impl Iterator> { + vec.into_iter().map(move |s| s.map_span(|span| span.offset(start))) +} + +/// A value with the span it corresponds to in the source code. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)] +pub struct Spanned { + /// The value. + pub v: T, + /// The corresponding span. + pub span: Span, +} + +impl Spanned { + /// Create a new instance from a value and its span. + pub fn new(v: T, span: Span) -> Spanned { + Spanned { v, span } + } + + /// Create a new instance from a value with the zero span. + pub fn zero(v: T) -> Spanned { + Spanned { v, span: Span::ZERO } + } + + /// Access the value. + pub fn value(self) -> T { + self.v + } + + /// Map the value using a function while keeping the span. + pub fn map(self, f: F) -> Spanned where F: FnOnce(T) -> V { + Spanned { v: f(self.v), span: self.span } + } + + /// Maps the span while keeping the value. + pub fn map_span(mut self, f: F) -> Spanned where F: FnOnce(Span) -> Span { + self.span = f(self.span); + self + } +} + +impl Debug for Spanned { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + self.v.fmt(f)?; + if f.alternate() { + f.write_str(" ")?; + self.span.fmt(f)?; + } + Ok(()) + } +} + +/// Locates a slice of source code. +#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)] +pub struct Span { + /// The inclusive start position. + pub start: Position, + /// The inclusive end position. + pub end: Position, +} + +impl Span { + /// The zero span. + pub const ZERO: Span = Span { start: Position::ZERO, end: Position::ZERO }; + + /// Create a new span from start and end positions. + pub fn new(start: Position, end: Position) -> Span { + Span { start, end } + } + + /// Create a new span with the earlier start and later end position. + pub fn merge(a: Span, b: Span) -> Span { + Span { + start: a.start.min(b.start), + end: a.end.max(b.end), + } + } + + /// Create a span including just a single position. + pub fn at(pos: Position) -> Span { + Span { start: pos, end: pos } + } + + /// Offset a span by a start position. + /// + /// This is, for example, used to translate error spans from function local + /// to global. + pub fn offset(self, start: Position) -> Span { + Span { + start: start + self.start, + end: start + self.end, + } + } +} + +impl Debug for Span { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "<{:?}-{:?}>", self.start, self.end) + } +} /// Zero-indexed line-column position in source code. #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize)] @@ -65,102 +173,3 @@ impl Debug for Position { write!(f, "{}:{}", self.line, self.column) } } - -/// Locates a slice of source code. -#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)] -pub struct Span { - /// The inclusive start position. - pub start: Position, - /// The inclusive end position. - pub end: Position, -} - -impl Span { - /// The zero span. - pub const ZERO: Span = Span { start: Position::ZERO, end: Position::ZERO }; - - /// Create a new span from start and end positions. - pub fn new(start: Position, end: Position) -> Span { - Span { start, end } - } - - /// Create a new span with the earlier start and later end position. - pub fn merge(a: Span, b: Span) -> Span { - Span { - start: a.start.min(b.start), - end: a.end.max(b.end), - } - } - - /// Create a span including just a single position. - pub fn at(pos: Position) -> Span { - Span { start: pos, end: pos } - } - - /// Offset a span by a start position. - /// - /// This is, for example, used to translate error spans from function local - /// to global. - pub fn offset(self, start: Position) -> Span { - Span { - start: start + self.start, - end: start + self.end, - } - } -} - -impl Debug for Span { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!(f, "({:?} -> {:?})", self.start, self.end) - } -} - -/// A value with the span it corresponds to in the source code. -#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)] -pub struct Spanned { - /// The value. - pub v: T, - /// The corresponding span. - pub span: Span, -} - -impl Spanned { - /// Create a new instance from a value and its span. - pub fn new(v: T, span: Span) -> Spanned { - Spanned { v, span } - } - - /// Access the value. - pub fn value(self) -> T { - self.v - } - - /// Map the value using a function while keeping the span. - pub fn map(self, f: F) -> Spanned where F: FnOnce(T) -> V { - Spanned { v: f(self.v), span: self.span } - } - - /// Maps the span while keeping the value. - pub fn map_span(mut self, f: F) -> Spanned where F: FnOnce(Span) -> Span { - self.span = f(self.span); - self - } -} - -impl Debug for Spanned { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - self.v.fmt(f)?; - f.write_str(" ")?; - self.span.fmt(f) - } -} - -/// A vector of spanned things. -pub type SpanVec = Vec>; - -/// [Offset](Span::offset) all spans in a vector of spanned things by a start -/// position. -pub fn offset_spans(start: Position, vec: SpanVec) -> impl Iterator> { - vec.into_iter() - .map(move |s| s.map_span(|span| span.offset(start))) -} diff --git a/src/syntax/test.rs b/src/syntax/test.rs index 465f475af..aaad5c127 100644 --- a/src/syntax/test.rs +++ b/src/syntax/test.rs @@ -1,17 +1,14 @@ use std::fmt::Debug; use super::func::FuncHeader; +use super::span::Spanned; use super::expr::{Expr, Tuple, NamedTuple, Object}; -use super::span::{Span, Spanned}; -use super::tokens::Token; use super::*; - -/// Check whether the expected and found results for the given source code -/// match by the comparison function, and print them out otherwise. -pub fn check(src: &str, exp: T, found: T, spans: bool) +/// Check whether the expected and found results are the same. +pub fn check(src: &str, exp: T, found: T, cmp_spans: bool) where T: Debug + PartialEq + SpanlessEq { - let cmp = if spans { PartialEq::eq } else { SpanlessEq::spanless_eq }; + let cmp = if cmp_spans { PartialEq::eq } else { SpanlessEq::spanless_eq }; if !cmp(&exp, &found) { println!("source: {:?}", src); println!("expected: {:#?}", exp); @@ -23,16 +20,25 @@ where T: Debug + PartialEq + SpanlessEq { /// Create a vector of optionally spanned expressions from a list description. /// /// # Examples -/// When you want to add span information to the items, the format is as -/// follows. /// ``` +/// // With spans /// spanned![(0:0, 0:5, "hello"), (0:5, 0:3, "world")] +/// +/// // Without spans: Implicit zero spans. +/// spanned!["hello", "world"] /// ``` -/// The span information can simply be omitted to create a vector with items -/// that are spanned with zero spans. -macro_rules! spanned { - (item ($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)) => ({ - #[allow(unused_imports)] +macro_rules! span_vec { + ($(($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)),* $(,)?) => { + (vec![$(span_item!(($sl:$sc, $el:$ec, $v))),*], true) + }; + + ($($v:expr),* $(,)?) => { + (vec![$(span_item!($v)),*], false) + }; +} + +macro_rules! span_item { + (($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)) => ({ use $crate::syntax::span::{Position, Span, Spanned}; Spanned { span: Span::new( @@ -43,22 +49,9 @@ macro_rules! spanned { } }); - (item $v:expr) => { - $crate::syntax::test::zspan($v) + ($v:expr) => { + $crate::syntax::span::Spanned::zero($v) }; - - (vec $(($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)),* $(,)?) => { - (vec![$(spanned![item ($sl:$sc, $el:$ec, $v)]),*], true) - }; - - (vec $($v:expr),* $(,)?) => { - (vec![$($crate::syntax::test::zspan($v)),*], false) - }; -} - -/// Span an element with a zero span. -pub fn zspan(v: T) -> Spanned { - Spanned { v, span: Span::ZERO } } function! { @@ -120,8 +113,8 @@ impl SpanlessEq for DebugFn { impl SpanlessEq for Expr { fn spanless_eq(&self, other: &Expr) -> bool { match (self, other) { - (Expr::NamedTuple(a), Expr::NamedTuple(b)) => a.spanless_eq(b), (Expr::Tuple(a), Expr::Tuple(b)) => a.spanless_eq(b), + (Expr::NamedTuple(a), Expr::NamedTuple(b)) => a.spanless_eq(b), (Expr::Object(a), Expr::Object(b)) => a.spanless_eq(b), (Expr::Neg(a), Expr::Neg(b)) => a.spanless_eq(&b), (Expr::Add(a1, a2), Expr::Add(b1, b2)) => a1.spanless_eq(&b1) && a2.spanless_eq(&b2), @@ -175,8 +168,7 @@ impl SpanlessEq for Box { } } -/// Implement `SpanlessEq` by just forwarding to `PartialEq`. -macro_rules! forward { +macro_rules! impl_through_partial_eq { ($type:ty) => { impl SpanlessEq for $type { fn spanless_eq(&self, other: &$type) -> bool { @@ -186,6 +178,8 @@ macro_rules! forward { }; } -forward!(String); -forward!(Token<'_>); -forward!(Decoration); +impl_through_partial_eq!(Token<'_>); + +// Implement for string and decoration to be able to compare feedback. +impl_through_partial_eq!(String); +impl_through_partial_eq!(Decoration); diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs index 0fc52f266..b1bf76e3e 100644 --- a/src/syntax/tokens.rs +++ b/src/syntax/tokens.rs @@ -532,6 +532,7 @@ pub fn is_identifier(string: &str) -> bool { #[cfg(test)] +#[allow(non_snake_case)] mod tests { use super::super::test::check; use super::*; @@ -552,31 +553,23 @@ mod tests { Slash, }; - #[allow(non_snake_case)] - fn Str(string: &'static str, terminated: bool) -> Token<'static> { - Token::ExprStr { string, terminated } - } - - #[allow(non_snake_case)] - fn Raw(raw: &'static str, terminated: bool) -> Token<'static> { - Token::Raw { raw, terminated } - } - /// Test whether the given string tokenizes into the given list of tokens. macro_rules! t { ($mode:expr, $source:expr => [$($tokens:tt)*]) => { - let (exp, spans) = spanned![vec $($tokens)*]; + let (exp, spans) = span_vec![$($tokens)*]; let found = Tokens::new(Position::ZERO, $source, $mode).collect::>(); check($source, exp, found, spans); } } - /// Write down a function token compactly. + fn Str(string: &str, terminated: bool) -> Token { Token::ExprStr { string, terminated } } + fn Raw(raw: &str, terminated: bool) -> Token { Token::Raw { raw, terminated } } + macro_rules! func { ($header:expr, Some($($tokens:tt)*), $terminated:expr) => { Function { header: $header, - body: Some(spanned![item $($tokens)*]), + body: Some(span_item!(($($tokens)*))), terminated: $terminated, } }; @@ -677,12 +670,12 @@ mod tests { fn tokenize_functions() { t!(Body, "a[f]" => [T("a"), func!("f", None, true)]); t!(Body, "[f]a" => [func!("f", None, true), T("a")]); - t!(Body, "\n\n[f][ ]" => [S(2), func!("f", Some((0:4, 0:5, " ")), true)]); - t!(Body, "abc [f][ ]a" => [T("abc"), S(0), func!("f", Some((0:4, 0:5, " ")), true), T("a")]); + t!(Body, "\n\n[f][ ]" => [S(2), func!("f", Some(0:4, 0:5, " "), true)]); + t!(Body, "abc [f][ ]a" => [T("abc"), S(0), func!("f", Some(0:4, 0:5, " "), true), T("a")]); t!(Body, "[f: [=][*]]" => [func!("f: [=][*]", None, true)]); - t!(Body, "[_][[,],]," => [func!("_", Some((0:4, 0:8, "[,],")), true), T(",")]); - t!(Body, "[=][=][=]" => [func!("=", Some((0:4, 0:5, "=")), true), func!("=", None, true)]); - t!(Body, "[=][[=][=][=]]" => [func!("=", Some((0:4, 0:13, "[=][=][=]")), true)]); + t!(Body, "[_][[,],]," => [func!("_", Some(0:4, 0:8, "[,],"), true), T(",")]); + t!(Body, "[=][=][=]" => [func!("=", Some(0:4, 0:5, "="), true), func!("=", None, true)]); + t!(Body, "[=][[=][=][=]]" => [func!("=", Some(0:4, 0:13, "[=][=][=]"), true)]); t!(Header, "[" => [func!("", None, false)]); t!(Header, "]" => [Invalid("]")]); } @@ -696,25 +689,25 @@ mod tests { t!(Body, "[f: `]" => [func!("f: `", None, true)]); // End of function with strings and carets in bodies - t!(Body, "[f][\"]" => [func!("f", Some((0:4, 0:5, "\"")), true)]); - t!(Body, r#"[f][\"]"# => [func!("f", Some((0:4, 0:6, r#"\""#)), true)]); - t!(Body, "[f][`]" => [func!("f", Some((0:4, 0:6, "`]")), false)]); - t!(Body, "[f][\\`]" => [func!("f", Some((0:4, 0:6, "\\`")), true)]); - t!(Body, "[f][`raw`]" => [func!("f", Some((0:4, 0:9, "`raw`")), true)]); - t!(Body, "[f][`raw]" => [func!("f", Some((0:4, 0:9, "`raw]")), false)]); - t!(Body, "[f][`raw]`]" => [func!("f", Some((0:4, 0:10, "`raw]`")), true)]); - t!(Body, "[f][`\\`]" => [func!("f", Some((0:4, 0:8, "`\\`]")), false)]); - t!(Body, "[f][`\\\\`]" => [func!("f", Some((0:4, 0:8, "`\\\\`")), true)]); + t!(Body, "[f][\"]" => [func!("f", Some(0:4, 0:5, "\""), true)]); + t!(Body, r#"[f][\"]"# => [func!("f", Some(0:4, 0:6, r#"\""#), true)]); + t!(Body, "[f][`]" => [func!("f", Some(0:4, 0:6, "`]"), false)]); + t!(Body, "[f][\\`]" => [func!("f", Some(0:4, 0:6, "\\`"), true)]); + t!(Body, "[f][`raw`]" => [func!("f", Some(0:4, 0:9, "`raw`"), true)]); + t!(Body, "[f][`raw]" => [func!("f", Some(0:4, 0:9, "`raw]"), false)]); + t!(Body, "[f][`raw]`]" => [func!("f", Some(0:4, 0:10, "`raw]`"), true)]); + t!(Body, "[f][`\\`]" => [func!("f", Some(0:4, 0:8, "`\\`]"), false)]); + t!(Body, "[f][`\\\\`]" => [func!("f", Some(0:4, 0:8, "`\\\\`"), true)]); // End of function with comments - t!(Body, "[f][/*]" => [func!("f", Some((0:4, 0:7, "/*]")), false)]); - t!(Body, "[f][/*`*/]" => [func!("f", Some((0:4, 0:9, "/*`*/")), true)]); + t!(Body, "[f][/*]" => [func!("f", Some(0:4, 0:7, "/*]"), false)]); + t!(Body, "[f][/*`*/]" => [func!("f", Some(0:4, 0:9, "/*`*/"), true)]); t!(Body, "[f: //]\n]" => [func!("f: //]\n", None, true)]); t!(Body, "[f: \"//]\n]" => [func!("f: \"//]\n]", None, false)]); // End of function with escaped brackets - t!(Body, "[f][\\]]" => [func!("f", Some((0:4, 0:6, "\\]")), true)]); - t!(Body, "[f][\\[]" => [func!("f", Some((0:4, 0:6, "\\[")), true)]); + t!(Body, "[f][\\]]" => [func!("f", Some(0:4, 0:6, "\\]"), true)]); + t!(Body, "[f][\\[]" => [func!("f", Some(0:4, 0:6, "\\["), true)]); } #[test] From 53ca5a7fc5829d4c5b1cffc6d5a5f1706f8ec3cd Mon Sep 17 00:00:00 2001 From: Laurenz Date: Mon, 27 Jul 2020 13:47:29 +0200 Subject: [PATCH 3/4] =?UTF-8?q?Refactor=20parser=20=F0=9F=9A=87?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/func.rs | 36 +- src/layout/model.rs | 2 +- src/lib.rs | 19 +- src/library/mod.rs | 4 +- src/library/page.rs | 4 +- src/library/spacing.rs | 6 +- src/macros.rs | 25 ++ src/syntax/func/mod.rs | 17 +- src/syntax/parsing.rs | 765 ++++++++++++++++++++--------------------- src/syntax/scope.rs | 22 +- src/syntax/span.rs | 11 +- src/syntax/test.rs | 4 +- src/syntax/tokens.rs | 24 +- 13 files changed, 468 insertions(+), 471 deletions(-) create mode 100644 src/macros.rs diff --git a/src/func.rs b/src/func.rs index 777e0b26b..e948d0499 100644 --- a/src/func.rs +++ b/src/func.rs @@ -1,9 +1,8 @@ //! Trait and prelude for custom functions. use crate::Pass; -use crate::syntax::ParseContext; -use crate::syntax::func::FuncHeader; -use crate::syntax::span::Spanned; +use crate::syntax::ParseState; +use crate::syntax::func::FuncCall; /// Types that are useful for creating your own functions. pub mod prelude { @@ -17,7 +16,6 @@ pub mod prelude { pub use crate::syntax::span::{Span, Spanned}; } - /// Parse a function from source code. pub trait ParseFunc { /// A metadata type whose value is passed into the function parser. This @@ -33,9 +31,8 @@ pub trait ParseFunc { /// Parse the header and body into this function given a context. fn parse( - header: FuncHeader, - body: Option>, - ctx: ParseContext, + header: FuncCall, + state: &ParseState, metadata: Self::Meta, ) -> Pass where Self: Sized; } @@ -53,8 +50,8 @@ pub trait ParseFunc { /// body: Option, /// } /// -/// parse(header, body, ctx, f) { -/// let body = body!(opt: body, ctx, f); +/// parse(header, body, state, f) { +/// let body = body!(opt: body, state, f); /// let hidden = header.args.pos.get::(&mut f.problems) /// .or_missing(&mut f.problems, header.name.span, "hidden") /// .unwrap_or(false); @@ -112,7 +109,7 @@ macro_rules! function { (@parse($name:ident, $meta:ty) parse( $header:ident, $body:ident, - $ctx:ident, + $state:ident, $feedback:ident, $metadata:ident ) $code:block $($r:tt)*) => { @@ -120,18 +117,18 @@ macro_rules! function { type Meta = $meta; fn parse( - #[allow(unused)] mut header: $crate::syntax::func::FuncHeader, - #[allow(unused)] $body: Option<$crate::syntax::span::Spanned<&str>>, - #[allow(unused)] $ctx: $crate::syntax::ParseContext, + #[allow(unused)] mut call: $crate::syntax::func::FuncCall, + #[allow(unused)] $state: &$crate::syntax::ParseState, #[allow(unused)] $metadata: Self::Meta, ) -> $crate::Pass where Self: Sized { let mut feedback = $crate::Feedback::new(); - #[allow(unused)] let $header = &mut header; + #[allow(unused)] let $header = &mut call.header; + #[allow(unused)] let $body = &mut call.body; #[allow(unused)] let $feedback = &mut feedback; let func = $code; - for arg in header.args.into_iter() { + for arg in call.header.args.into_iter() { error!(@feedback, arg.span, "unexpected argument"); } @@ -167,21 +164,20 @@ macro_rules! function { /// Parse the body of a function. /// /// - If the function does not expect a body, use `body!(nope: body, feedback)`. -/// - If the function can have a body, use `body!(opt: body, ctx, feedback, +/// - If the function can have a body, use `body!(opt: body, state, feedback, /// decos)`. /// /// # Arguments /// - The `$body` should be of type `Option>`. -/// - The `$ctx` is the [`ParseContext`](crate::syntax::ParseContext) to use for -/// parsing. +/// - The `$state` is the parse state to use. /// - The `$feedback` should be a mutable references to a /// [`Feedback`](crate::Feedback) struct which is filled with the feedback /// information arising from parsing. #[macro_export] macro_rules! body { - (opt: $body:expr, $ctx:expr, $feedback:expr) => ({ + (opt: $body:expr, $state:expr, $feedback:expr) => ({ $body.map(|body| { - let parsed = $crate::syntax::parse(body.span.start, body.v, $ctx); + let parsed = $crate::syntax::parse(body.v, body.span.start, $state); $feedback.extend(parsed.feedback); parsed.output }) diff --git a/src/layout/model.rs b/src/layout/model.rs index 3f1c9e7a0..91d21037e 100644 --- a/src/layout/model.rs +++ b/src/layout/model.rs @@ -142,7 +142,7 @@ impl<'a> ModelLayouter<'a> { }).await; // Add the errors generated by the model to the error list. - self.feedback.extend_offset(model.span.start, layouted.feedback); + self.feedback.extend_offset(layouted.feedback, model.span.start); for command in layouted.output { self.execute_command(command, model.span).await; diff --git a/src/lib.rs b/src/lib.rs index b140f47da..7f1752796 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,7 +30,7 @@ use toddle::query::{FontProvider, FontIndex, FontDescriptor}; use crate::problem::Problems; use crate::layout::MultiLayout; use crate::style::{LayoutStyle, PageStyle, TextStyle}; -use crate::syntax::{SyntaxModel, Scope, Decoration, ParseContext, parse}; +use crate::syntax::{SyntaxModel, Scope, Decoration, ParseState, parse}; use crate::syntax::span::{Position, SpanVec, offset_spans}; @@ -42,6 +42,8 @@ macro_rules! pub_use_mod { }; } +#[macro_use] +mod macros; #[macro_use] pub mod problem; pub mod export; @@ -57,14 +59,13 @@ pub mod syntax; /// Transforms source code into typesetted layouts. /// /// A typesetter can be configured through various methods. -#[derive(Debug)] pub struct Typesetter { /// The font loader shared by all typesetting processes. loader: GlobalFontLoader, /// The base layouting style. style: LayoutStyle, - /// The standard library scope. - scope: Scope, + /// The base parser state. + parse_state: ParseState, /// Whether to render debug boxes. debug: bool, } @@ -84,7 +85,7 @@ impl Typesetter { Typesetter { loader: RefCell::new(FontLoader::new(provider)), style: LayoutStyle::default(), - scope: Scope::with_std(), + parse_state: ParseState { scope: Scope::with_std() }, debug: false, } } @@ -111,7 +112,7 @@ impl Typesetter { /// Parse source code into a syntax tree. pub fn parse(&self, src: &str) -> Pass { - parse(Position::ZERO, src, ParseContext { scope: &self.scope }) + parse(src, Position::ZERO, &self.parse_state) } /// Layout a syntax tree and return the produced layout. @@ -204,9 +205,9 @@ impl Feedback { /// Add more feedback whose spans are local and need to be offset by an /// `offset` to be correct for this feedbacks context. - pub fn extend_offset(&mut self, offset: Position, other: Feedback) { - self.problems.extend(offset_spans(offset, other.problems)); - self.decos.extend(offset_spans(offset, other.decos)); + pub fn extend_offset(&mut self, other: Feedback, offset: Position) { + self.problems.extend(offset_spans(other.problems, offset)); + self.decos.extend(offset_spans(other.decos, offset)); } } diff --git a/src/library/mod.rs b/src/library/mod.rs index c8ca374df..e1aa4ac20 100644 --- a/src/library/mod.rs +++ b/src/library/mod.rs @@ -53,10 +53,10 @@ function! { body: Option, } - parse(header, body, ctx, f) { + parse(header, body, state, f) { header.args.pos.items.clear(); header.args.key.pairs.clear(); - ValFunc { body: body!(opt: body, ctx, f) } + ValFunc { body: body!(opt: body, state, f) } } layout(self, ctx, f) { diff --git a/src/library/page.rs b/src/library/page.rs index 4d92ea919..1e782f8f7 100644 --- a/src/library/page.rs +++ b/src/library/page.rs @@ -12,7 +12,7 @@ function! { flip: bool, } - parse(header, body, ctx, f) { + parse(header, body, state, f) { body!(nope: body, f); PageSizeFunc { paper: header.args.pos.get::(&mut f.problems), @@ -50,7 +50,7 @@ function! { padding: PaddingMap, } - parse(header, body, ctx, f) { + parse(header, body, state, f) { body!(nope: body, f); PageMarginsFunc { padding: PaddingMap::parse(&mut f.problems, &mut header.args), diff --git a/src/library/spacing.rs b/src/library/spacing.rs index 8d9c46aa7..adca20afd 100644 --- a/src/library/spacing.rs +++ b/src/library/spacing.rs @@ -46,9 +46,9 @@ function! { type Meta = ContentKind; - parse(header, body, ctx, f, meta) { + parse(header, body, state, f, meta) { ContentSpacingFunc { - body: body!(opt: body, ctx, f), + body: body!(opt: body, state, f), content: meta, spacing: header.args.pos.get::(&mut f.problems) .map(|num| num as f32) @@ -84,7 +84,7 @@ function! { type Meta = Option; - parse(header, body, ctx, f, meta) { + parse(header, body, state, f, meta) { body!(nope: body, f); SpacingFunc { spacing: if let Some(axis) = meta { diff --git a/src/macros.rs b/src/macros.rs new file mode 100644 index 000000000..8ba326503 --- /dev/null +++ b/src/macros.rs @@ -0,0 +1,25 @@ +#![allow(unused)] + +/// Unwrap the result if it is `Ok(T)` or evaluate `$or` if it is `Err(_)`. +/// This fits use cases the `?`-operator does not cover, like: +/// ``` +/// try_or!(result, continue); +/// ``` +macro_rules! try_or { + ($result:expr, $or:expr $(,)?) => { + match $result { + Ok(v) => v, + Err(_) => { $or } + } + }; +} + +/// Unwrap the option if it is `Some(T)` or evaluate `$or` if it is `None`. +macro_rules! try_opt_or { + ($option:expr, $or:expr $(,)?) => { + match $option { + Some(v) => v, + None => { $or } + } + }; +} diff --git a/src/syntax/func/mod.rs b/src/syntax/func/mod.rs index d379b4071..4228488d9 100644 --- a/src/syntax/func/mod.rs +++ b/src/syntax/func/mod.rs @@ -9,22 +9,23 @@ pub_use_mod!(maps); pub_use_mod!(keys); pub_use_mod!(values); +/// An invocation of a function. +#[derive(Debug, Clone, PartialEq)] +pub struct FuncCall<'s> { + pub header: FuncHeader, + /// The body as a raw string containing what's inside of the brackets. + pub body: Option>, +} -/// The parsed header of a function. +/// The parsed header of a function (everything in the first set of brackets). #[derive(Debug, Clone, PartialEq)] pub struct FuncHeader { - /// The function name, that is: - /// ```typst - /// [box: w=5cm] - /// ^^^ - /// ``` pub name: Spanned, - /// The arguments passed to the function. pub args: FuncArgs, } /// The positional and keyword arguments passed to a function. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Default, Clone, PartialEq)] pub struct FuncArgs { /// The positional arguments. pub pos: Tuple, diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs index 89d4f0fd2..ece09a866 100644 --- a/src/syntax/parsing.rs +++ b/src/syntax/parsing.rs @@ -1,52 +1,45 @@ //! Parsing of source code into syntax models. -use std::iter::FromIterator; use std::str::FromStr; -use crate::{Pass, Feedback}; -use super::func::{FuncHeader, FuncArgs, FuncArg}; use super::expr::*; -use super::scope::Scope; +use super::func::{FuncCall, FuncHeader, FuncArgs, FuncArg}; use super::span::{Position, Span, Spanned}; -use super::tokens::{Token, Tokens, TokenizationMode}; use super::*; - -/// The context for parsing. -#[derive(Debug, Copy, Clone)] -pub struct ParseContext<'a> { - /// The scope containing function definitions. - pub scope: &'a Scope, +/// The state which can influence how a string of source code is parsed. +/// +/// Parsing is pure - when passed in the same state and source code, the output +/// must be the same. +pub struct ParseState { + /// The scope containing all function definitions. + pub scope: Scope, } -/// Parse source code into a syntax model. +/// Parse a string of source code. /// -/// All errors and decorations are offset by the `start` position. -pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Pass { +/// All spans in the resulting model and feedback are offset by the given +/// `offset` position. This is used to make spans of a function body relative to +/// the start of the function as a whole as opposed to the start of the +/// function's body. +pub fn parse(src: &str, offset: Position, state: &ParseState) -> Pass { let mut model = SyntaxModel::new(); let mut feedback = Feedback::new(); - // We always start in body mode. The header tokenization mode is only used - // in the `FuncParser`. - let mut tokens = Tokens::new(start, src, TokenizationMode::Body); - - while let Some(token) = tokens.next() { + for token in Tokens::new(src, offset, TokenMode::Body) { let span = token.span; - let node = match token.v { - Token::LineComment(_) | Token::BlockComment(_) => continue, - - // Only at least two newlines mean a _real_ newline indicating a - // paragraph break. + // Starting from two newlines counts as a paragraph break, a single + // newline not. Token::Space(newlines) => if newlines >= 2 { Node::Parbreak } else { Node::Space - }, + } Token::Function { header, body, terminated } => { - let parsed = FuncParser::new(header, body, ctx).parse(); - feedback.extend_offset(span.start, parsed.feedback); + let parsed = FuncParser::new(header, body, state).parse(); + feedback.extend_offset(parsed.feedback, span.start); if !terminated { error!(@feedback, Span::at(span.end), "expected closing bracket"); @@ -55,9 +48,9 @@ pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Pass parsed.output } - Token::Star => Node::ToggleBolder, + Token::Star => Node::ToggleBolder, Token::Underscore => Node::ToggleItalic, - Token::Backslash => Node::Linebreak, + Token::Backslash => Node::Linebreak, Token::Raw { raw, terminated } => { if !terminated { @@ -69,103 +62,101 @@ pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Pass Token::Text(text) => Node::Text(text.to_string()), - other => { - error!(@feedback, span, "unexpected {}", other.name()); + Token::LineComment(_) | Token::BlockComment(_) => continue, + unexpected => { + error!(@feedback, span, "unexpected {}", unexpected.name()); continue; } }; - model.add(Spanned { v: node, span: token.span }); + model.add(Spanned::new(node, span)); } Pass::new(model, feedback) } -/// Performs the function parsing. struct FuncParser<'s> { - ctx: ParseContext<'s>, - feedback: Feedback, - + state: &'s ParseState, /// ```typst /// [tokens][body] /// ^^^^^^ /// ``` tokens: Tokens<'s>, peeked: Option>>>, - /// The spanned body string if there is a body. /// ```typst /// [tokens][body] /// ^^^^ /// ``` body: Option>, + feedback: Feedback, } impl<'s> FuncParser<'s> { - /// Create a new function parser. fn new( header: &'s str, body: Option>, - ctx: ParseContext<'s> + state: &'s ParseState, ) -> FuncParser<'s> { FuncParser { - ctx, - feedback: Feedback::new(), - tokens: Tokens::new(Position::new(0, 1), header, TokenizationMode::Header), + state, + // Start at column 1 because the opening bracket is also part of + // the function, but not part of the `header` string. + tokens: Tokens::new(header, Position::new(0, 1), TokenMode::Header), peeked: None, body, + feedback: Feedback::new(), } } - /// Do the parsing. fn parse(mut self) -> Pass { - let parsed = if let Some(header) = self.parse_func_header() { + let (parser, header) = if let Some(header) = self.parse_func_header() { let name = header.name.v.as_str(); - let (parser, deco) = match self.ctx.scope.get_parser(name) { - // A valid function. - Ok(parser) => (parser, Decoration::ValidFuncName), + let (parser, deco) = match self.state.scope.get_parser(name) { + // The function exists in the scope. + Some(parser) => (parser, Decoration::ValidFuncName), - // The fallback parser was returned. Invalid function. - Err(parser) => { + // The function does not exist in the scope. The parser that is + // returned here is a fallback parser which exists to make sure + // the content of the function is not totally dropped (on a best + // effort basis). + None => { error!(@self.feedback, header.name.span, "unknown function"); + let parser = self.state.scope.get_fallback_parser(); (parser, Decoration::InvalidFuncName) } }; self.feedback.decos.push(Spanned::new(deco, header.name.span)); - - parser(header, self.body, self.ctx) + (parser, header) } else { - let default = FuncHeader { - name: Spanned::new(Ident("".to_string()), Span::ZERO), + // Parse the body with the fallback parser even when the header is + // completely unparsable. + let parser = self.state.scope.get_fallback_parser(); + let header = FuncHeader { + name: Spanned::new(Ident(String::new()), Span::ZERO), args: FuncArgs::new(), }; - - // Use the fallback function such that the body is still rendered - // even if the header is completely unparsable. - self.ctx.scope.get_fallback_parser()(default, self.body, self.ctx) + (parser, header) }; - self.feedback.extend(parsed.feedback); + let call = FuncCall { header, body: self.body }; + let parsed = parser(call, self.state); + self.feedback.extend(parsed.feedback); Pass::new(Node::Model(parsed.output), self.feedback) } - /// Parse the header tokens. fn parse_func_header(&mut self) -> Option { - let start = self.pos(); - self.skip_whitespace(); + let after_bracket = self.pos(); - let name = match self.parse_ident() { - Some(ident) => ident, - None => { - let other = self.eat(); - self.expected_found_or_at("identifier", other, start); - return None; - } - }; + self.skip_white(); + let name = try_opt_or!(self.parse_ident(), { + self.expected_found_or_at("function name", after_bracket); + return None; + }); - self.skip_whitespace(); + self.skip_white(); let args = match self.eat().map(Spanned::value) { Some(Token::Colon) => self.parse_func_args(), Some(_) => { @@ -178,58 +169,74 @@ impl<'s> FuncParser<'s> { Some(FuncHeader { name, args }) } - /// Parse the argument list between colons and end of the header. fn parse_func_args(&mut self) -> FuncArgs { - // Parse a collection until the token is `None`, that is, the end of the - // header. - self.parse_collection(None, |p| { - // If we have an identifier we might have a keyword argument, - // otherwise its for sure a postional argument. - if let Some(ident) = p.parse_ident() { - // This could still be a named tuple - if let Some(Token::LeftParen) = p.peekv() { - let tuple = p.parse_named_tuple(ident); - return Ok(tuple.map(|t| FuncArg::Pos(Expr::NamedTuple(t)))); - } + let mut args = FuncArgs::new(); + loop { + self.skip_white(); + if self.eof() { + break; + } - p.skip_whitespace(); + let arg = if let Some(ident) = self.parse_ident() { + self.skip_white(); - if let Some(Token::Equals) = p.peekv() { - p.eat(); - p.skip_whitespace(); + // This could be a keyword argument, or a positional argument of + // type named tuple or identifier. + if self.check_eat(Token::Equals).is_some() { + self.skip_white(); - // Semantic highlighting for argument keys. - p.feedback.decos.push( - Spanned::new(Decoration::ArgumentKey, ident.span)); + let key = ident; + self.feedback.decos.push( + Spanned::new(Decoration::ArgumentKey, key.span) + ); - let value = p.parse_expr().ok_or(("value", None))?; + let value = try_opt_or!(self.parse_expr(), { + self.expected("value"); + continue; + }); - // Add a keyword argument. - let span = Span::merge(ident.span, value.span); - let pair = Pair { key: ident, value }; - Ok(Spanned::new(FuncArg::Key(pair), span)) + let span = Span::merge(key.span, value.span); + let arg = FuncArg::Key(Pair { key, value }); + Spanned::new(arg, span) + } else if self.check(Token::LeftParen) { + let tuple = self.parse_named_tuple(ident); + tuple.map(|tup| FuncArg::Pos(Expr::NamedTuple(tup))) } else { - // Add a positional argument because there was no equals - // sign after the identifier that could have been a key. - Ok(ident.map(|id| FuncArg::Pos(Expr::Ident(id)))) + ident.map(|id| FuncArg::Pos(Expr::Ident(id))) } } else { - // Add a positional argument because we haven't got an - // identifier that could be an argument key. - let value = p.parse_expr().ok_or(("argument", None))?; - Ok(value.map(|expr| FuncArg::Pos(expr))) + // It's a positional argument. + try_opt_or!(self.parse_expr(), { + self.expected("argument"); + continue; + }).map(|expr| FuncArg::Pos(expr)) + }; + + let behind_arg = arg.span.end; + args.add(arg); + + self.skip_white(); + if self.eof() { + break; } - }).v + + self.expect_at(Token::Comma, behind_arg); + } + args + } +} + +// Parsing expressions and values +impl FuncParser<'_> { + fn parse_ident(&mut self) -> Option> { + self.peek().and_then(|token| match token.v { + Token::ExprIdent(id) => self.eat_span(Ident(id.to_string())), + _ => None, + }) } - /// Parse an expression which may contain math operands. For this, this - /// method looks for operators in descending order of associativity, i.e. we - /// first drill down to find all negations, brackets and tuples, the next - /// level, we look for multiplication and division and here finally, for - /// addition and subtraction. fn parse_expr(&mut self) -> Option> { - let o1 = self.parse_term()?; - self.parse_binop(o1, "summand", Self::parse_expr, |token| match token { + self.parse_binops("summand", Self::parse_term, |token| match token { Token::Plus => Some(Expr::Add), Token::Hyphen => Some(Expr::Sub), _ => None, @@ -237,60 +244,57 @@ impl<'s> FuncParser<'s> { } fn parse_term(&mut self) -> Option> { - let o1 = self.parse_factor()?; - self.parse_binop(o1, "factor", Self::parse_term, |token| match token { + self.parse_binops("factor", Self::parse_factor, |token| match token { Token::Star => Some(Expr::Mul), Token::Slash => Some(Expr::Div), _ => None, }) } - fn parse_binop( + /// Parse expression of the form ` ( )*`. + fn parse_binops( &mut self, - o1: Spanned, operand_name: &str, - parse_operand: F, - parse_op: G, - ) -> Option> - where - F: FnOnce(&mut Self) -> Option>, - G: FnOnce(Token) -> Option>, Box>) -> Expr>, - { - self.skip_whitespace(); + mut parse_operand: impl FnMut(&mut Self) -> Option>, + mut parse_op: impl FnMut(Token) -> Option< + fn(Box>, Box>) -> Expr + >, + ) -> Option> { + let mut left = parse_operand(self)?; - if let Some(next) = self.peek() { - if let Some(binop) = parse_op(next.v) { + self.skip_white(); + while let Some(token) = self.peek() { + if let Some(op) = parse_op(token.v) { self.eat(); - self.skip_whitespace(); + self.skip_white(); - if let Some(o2) = parse_operand(self) { - let span = Span::merge(o1.span, o2.span); - let expr = binop(Box::new(o1), Box::new(o2)); - return Some(Spanned::new(expr, span)); - } else { - error!( - @self.feedback, Span::merge(next.span, o1.span), - "missing right {}", operand_name, - ); + if let Some(right) = parse_operand(self) { + let span = Span::merge(left.span, right.span); + let v = op(Box::new(left), Box::new(right)); + left = Spanned::new(v, span); + self.skip_white(); + continue; } + + error!( + @self.feedback, Span::merge(left.span, token.span), + "missing right {}", operand_name, + ); } + break; } - Some(o1) + Some(left) } - /// Parse expressions that are of the form value or -value. fn parse_factor(&mut self) -> Option> { - let first = self.peek()?; - if first.v == Token::Hyphen { - self.eat(); - self.skip_whitespace(); - - if let Some(factor) = self.parse_value() { - let span = Span::merge(first.span, factor.span); - Some(Spanned::new(Expr::Neg(Box::new(factor)), span)) + if let Some(hyph) = self.check_eat(Token::Hyphen) { + self.skip_white(); + if let Some(value) = self.parse_value() { + let span = Span::merge(hyph.span, value.span); + Some(Spanned::new(Expr::Neg(Box::new(value)), span)) } else { - error!(@self.feedback, first.span, "dangling minus"); + error!(@self.feedback, hyph.span, "dangling minus"); None } } else { @@ -299,283 +303,247 @@ impl<'s> FuncParser<'s> { } fn parse_value(&mut self) -> Option> { - let first = self.peek()?; - macro_rules! take { - ($v:expr) => ({ self.eat(); Spanned { v: $v, span: first.span } }); - } - - Some(match first.v { - Token::ExprIdent(i) => { - let name = take!(Ident(i.to_string())); - - // This could be a named tuple or an identifier - if let Some(Token::LeftParen) = self.peekv() { - self.parse_named_tuple(name).map(|t| Expr::NamedTuple(t)) + let Spanned { v: token, span } = self.peek()?; + match token { + // This could be a named tuple or an identifier. + Token::ExprIdent(id) => { + let name = Spanned::new(Ident(id.to_string()), span); + self.eat(); + self.skip_white(); + Some(if self.check(Token::LeftParen) { + self.parse_named_tuple(name).map(|tup| Expr::NamedTuple(tup)) } else { - name.map(|i| Expr::Ident(i)) - } - }, - Token::ExprStr { string, terminated } => { - if !terminated { - self.expected_at("quote", first.span.end); - } - - take!(Expr::Str(unescape_string(string))) + name.map(|id| Expr::Ident(id)) + }) } - Token::ExprNumber(n) => take!(Expr::Number(n)), - Token::ExprSize(s) => take!(Expr::Size(s)), - Token::ExprBool(b) => take!(Expr::Bool(b)), + Token::ExprStr { string, terminated } => { + if !terminated { + self.expected_at("quote", span.end); + } + self.eat_span(Expr::Str(unescape_string(string))) + } + + Token::ExprNumber(n) => self.eat_span(Expr::Number(n)), + Token::ExprSize(s) => self.eat_span(Expr::Size(s)), + Token::ExprBool(b) => self.eat_span(Expr::Bool(b)), Token::ExprHex(s) => { if let Ok(color) = RgbaColor::from_str(s) { - take!(Expr::Color(color)) + self.eat_span(Expr::Color(color)) } else { - // Heal color by assuming black - error!(@self.feedback, first.span, "invalid color"); - take!(Expr::Color(RgbaColor::new_healed(0, 0, 0, 255))) + // Heal color by assuming black. + error!(@self.feedback, span, "invalid color"); + let healed = RgbaColor::new_healed(0, 0, 0, 255); + self.eat_span(Expr::Color(healed)) } }, + // This could be a tuple or a parenthesized expression. We parse as + // a tuple in any case and coerce the tuple into a value if it is + // coercable (length 1 and no trailing comma). Token::LeftParen => { - let (mut tuple, can_be_coerced) = self.parse_tuple(); - // Coerce 1-tuple into value - if can_be_coerced && tuple.v.items.len() > 0 { - tuple.v.items.pop().expect("length is at least one") + let (mut tuple, coercable) = self.parse_tuple(); + Some(if coercable { + tuple.v.items.pop().expect("tuple is coercable") } else { - tuple.map(|t| Expr::Tuple(t)) - } - }, - Token::LeftBrace => self.parse_object().map(|o| Expr::Object(o)), + tuple.map(|tup| Expr::Tuple(tup)) + }) + } + Token::LeftBrace => { + Some(self.parse_object().map(|obj| Expr::Object(obj))) + } - _ => return None, - }) + _ => None, + } } - /// Parse a tuple expression: `(, ...)`. The boolean in the return - /// values showes whether the tuple can be coerced into a single value. - fn parse_tuple(&mut self) -> (Spanned, bool) { - let token = self.eat(); - debug_assert_eq!(token.map(Spanned::value), Some(Token::LeftParen)); - - // Parse a collection until a right paren appears and complain about - // missing a `value` when an invalid token is encoutered. - self.parse_collection_comma_aware(Some(Token::RightParen), - |p| p.parse_expr().ok_or(("value", None))) - } - - /// Parse a tuple expression: `name(, ...)` with a given identifier. fn parse_named_tuple(&mut self, name: Spanned) -> Spanned { let tuple = self.parse_tuple().0; let span = Span::merge(name.span, tuple.span); Spanned::new(NamedTuple::new(name, tuple), span) } - /// Parse an object expression: `{ : , ... }`. - fn parse_object(&mut self) -> Spanned { - let token = self.eat(); - debug_assert_eq!(token.map(Spanned::value), Some(Token::LeftBrace)); - - // Parse a collection until a right brace appears. - self.parse_collection(Some(Token::RightBrace), |p| { - // Expect an identifier as the key. - let key = p.parse_ident().ok_or(("key", None))?; - - // Expect a colon behind the key (only separated by whitespace). - let behind_key = p.pos(); - p.skip_whitespace(); - if p.peekv() != Some(Token::Colon) { - return Err(("colon", Some(behind_key))); - } - - p.eat(); - p.skip_whitespace(); - - // Semantic highlighting for object keys. - p.feedback.decos.push( - Spanned::new(Decoration::ObjectKey, key.span)); - - let value = p.parse_expr().ok_or(("value", None))?; - - let span = Span::merge(key.span, value.span); - Ok(Spanned::new(Pair { key, value }, span)) - }) - } - - /// Parse a comma-separated collection where each item is parsed through - /// `parse_item` until the `end` token is met. - fn parse_collection( - &mut self, - end: Option, - parse_item: F - ) -> Spanned - where - C: FromIterator>, - F: FnMut(&mut Self) -> Result, (&'static str, Option)>, - { - self.parse_collection_comma_aware(end, parse_item).0 - } - - /// Parse a comma-separated collection where each item is parsed through - /// `parse_item` until the `end` token is met. The first item in the return - /// tuple is the collection, the second item indicates whether the - /// collection can be coerced into a single item (i.e. no comma appeared). - fn parse_collection_comma_aware( - &mut self, - end: Option, - mut parse_item: F - ) -> (Spanned, bool) - where - C: FromIterator>, - F: FnMut(&mut Self) -> Result, (&'static str, Option)>, - { + /// The boolean tells you whether the tuple can be coerced into a value + /// (this is the case when it's length 1 and has no trailing comma). + fn parse_tuple(&mut self) -> (Spanned, bool) { let start = self.pos(); - let mut can_be_coerced = true; + self.assert(Token::LeftParen); - // Parse the comma separated items. - let collection = std::iter::from_fn(|| { - self.skip_whitespace(); - let peeked = self.peekv(); - - // We finished as expected. - if peeked == end { - self.eat(); - return None; + let mut tuple = Tuple::new(); + let mut commaless = true; + loop { + self.skip_white(); + if self.eof() || self.check(Token::RightParen) { + break; } - // We finished without the expected end token (which has to be a - // `Some` value at this point since otherwise we would have already - // returned in the previous case). - if peeked == None { - self.eat(); - self.expected_at(end.unwrap().name(), self.pos()); - return None; + let expr = try_opt_or!(self.parse_expr(), { + self.expected("value"); + continue; + }); + + let behind_expr = expr.span.end; + tuple.add(expr); + + self.skip_white(); + if self.eof() || self.check(Token::RightParen) { + break; } - // Try to parse a collection item. - match parse_item(self) { - Ok(item) => { - // Expect a comma behind the item (only separated by - // whitespace). - self.skip_whitespace(); - match self.peekv() { - Some(Token::Comma) => { - can_be_coerced = false; - self.eat(); - } - t @ Some(_) if t != end => { - can_be_coerced = false; - self.expected_at("comma", item.span.end); - }, - _ => {} - } - - return Some(Some(item)); - } - - // The item parser expected something different at either some - // given position or instead of the currently peekable token. - Err((expected, Some(pos))) => self.expected_at(expected, pos), - Err((expected, None)) => { - let token = self.peek(); - if token.map(Spanned::value) != end { - self.eat(); - } - self.expected_found_or_at(expected, token, self.pos()); - } - } - - Some(None) - }).filter_map(|x| x).collect(); + self.expect_at(Token::Comma, behind_expr); + commaless = false; + } + self.expect(Token::RightParen); let end = self.pos(); - (Spanned::new(collection, Span { start, end }), can_be_coerced) + let coercable = commaless && !tuple.items.is_empty(); + + (Spanned::new(tuple, Span::new(start, end)), coercable) } - /// Try to parse an identifier and do nothing if the peekable token is no - /// identifier. - fn parse_ident(&mut self) -> Option> { - match self.peek() { - Some(Spanned { v: Token::ExprIdent(s), span }) => { - self.eat(); - Some(Spanned { v: Ident(s.to_string()), span }) + fn parse_object(&mut self) -> Spanned { + let start = self.pos(); + self.assert(Token::LeftBrace); + + let mut object = Object::new(); + loop { + self.skip_white(); + if self.eof() || self.check(Token::RightBrace) { + break; } - _ => None + + let key = try_opt_or!(self.parse_ident(), { + self.expected("key"); + continue; + }); + + let after_key = self.pos(); + self.skip_white(); + if !self.expect_at(Token::Colon, after_key) { + continue; + } + + self.feedback.decos.push( + Spanned::new(Decoration::ObjectKey, key.span) + ); + + self.skip_white(); + let value = try_opt_or!(self.parse_expr(), { + self.expected("value"); + continue; + }); + + let behind_value = value.span.end; + let span = Span::merge(key.span, value.span); + object.add(Spanned::new(Pair { key, value }, span)); + + self.skip_white(); + if self.eof() || self.check(Token::RightBrace) { + break; + } + + self.expect_at(Token::Comma, behind_value); + } + + self.expect(Token::RightBrace); + let end = self.pos(); + + Spanned::new(object, Span::new(start, end)) + } +} + +// Error handling +impl FuncParser<'_> { + fn expect(&mut self, token: Token<'_>) -> bool { + if self.check(token) { + self.eat(); + true + } else { + self.expected(token.name()); + false } } - /// Skip all whitespace/comment tokens. - fn skip_whitespace(&mut self) { - self.eat_until(|t| match t { - Token::Space(_) | Token::LineComment(_) | - Token::BlockComment(_) => false, - _ => true, - }, false) + fn expect_at(&mut self, token: Token<'_>, pos: Position) -> bool { + if self.check(token) { + self.eat(); + true + } else { + self.expected_at(token.name(), pos); + false + } } - /// Add an error about an expected `thing` which was not found, showing - /// what was found instead. - fn expected_found(&mut self, thing: &str, found: Spanned) { - error!( - @self.feedback, found.span, - "expected {}, found {}", thing, found.v.name(), - ); + fn expected(&mut self, thing: &str) { + if let Some(found) = self.eat() { + error!( + @self.feedback, found.span, + "expected {}, found {}", thing, found.v.name(), + ); + } else { + error!(@self.feedback, Span::at(self.pos()), "expected {}", thing); + } } - /// Add an error about an `thing` which was expected but not found at the - /// given position. fn expected_at(&mut self, thing: &str, pos: Position) { error!(@self.feedback, Span::at(pos), "expected {}", thing); } - /// Add a expected-found-error if `found` is `Some` and an expected-error - /// otherwise. - fn expected_found_or_at( - &mut self, - thing: &str, - found: Option>, - pos: Position - ) { - match found { - Some(found) => self.expected_found(thing, found), - None => self.expected_at(thing, pos), + fn expected_found_or_at(&mut self, thing: &str, pos: Position) { + if self.eof() { + self.expected_at(thing, pos) + } else { + self.expected(thing); } } +} - /// Consume tokens until the function returns true and only consume the last - /// token if instructed to so by `eat_match`. - fn eat_until(&mut self, mut f: F, eat_match: bool) - where F: FnMut(Token<'s>) -> bool { - while let Some(token) = self.peek() { - if f(token.v) { - if eat_match { - self.eat(); - } - break; +// Parsing primitives +impl<'s> FuncParser<'s> { + fn skip_white(&mut self) { + loop { + match self.peek().map(Spanned::value) { + Some(Token::Space(_)) + | Some(Token::LineComment(_)) + | Some(Token::BlockComment(_)) => { self.eat(); } + _ => break, } - - self.eat(); } } - /// Consume and return the next token. fn eat(&mut self) -> Option>> { - self.peeked.take() - .unwrap_or_else(|| self.tokens.next()) + self.peeked.take().unwrap_or_else(|| self.tokens.next()) + } + + fn eat_span(&mut self, v: T) -> Option> { + self.eat().map(|spanned| spanned.map(|_| v)) } - /// Peek at the next token without consuming it. fn peek(&mut self) -> Option>> { - let iter = &mut self.tokens; - *self.peeked.get_or_insert_with(|| iter.next()) + let tokens = &mut self.tokens; + *self.peeked.get_or_insert_with(|| tokens.next()) } - /// Peek at the unspanned value of the next token. - fn peekv(&mut self) -> Option> { - self.peek().map(Spanned::value) + fn assert(&mut self, token: Token<'_>) { + assert!(self.check_eat(token).is_some()); + } + + fn check(&mut self, token: Token<'_>) -> bool { + self.peek().map(Spanned::value) == Some(token) + } + + fn check_eat(&mut self, token: Token<'_>) -> Option>> { + if self.check(token) { + self.eat() + } else { + None + } + } + + fn eof(&mut self) -> bool { + self.peek().is_none() } - /// The position at the end of the last eaten token / start of the peekable - /// token. fn pos(&self) -> Position { self.peeked.flatten() .map(|s| s.span.start) @@ -583,64 +551,62 @@ impl<'s> FuncParser<'s> { } } -/// Unescape a string: `the string is \"this\"` => `the string is "this"`. fn unescape_string(string: &str) -> String { - let mut s = String::with_capacity(string.len()); let mut iter = string.chars(); + let mut out = String::with_capacity(string.len()); while let Some(c) = iter.next() { if c == '\\' { match iter.next() { - Some('\\') => s.push('\\'), - Some('"') => s.push('"'), - Some('n') => s.push('\n'), - Some('t') => s.push('\t'), - Some(c) => { s.push('\\'); s.push(c); } - None => s.push('\\'), + Some('\\') => out.push('\\'), + Some('"') => out.push('"'), + Some('n') => out.push('\n'), + Some('t') => out.push('\t'), + Some(c) => { out.push('\\'); out.push(c); } + None => out.push('\\'), } } else { - s.push(c); + out.push(c); } } - s + out } -/// Unescape raw markup into lines. +/// Unescape raw markup and split it into into lines. fn unescape_raw(raw: &str) -> Vec { - let mut lines = Vec::new(); - let mut s = String::new(); let mut iter = raw.chars().peekable(); + let mut line = String::new(); + let mut lines = Vec::new(); while let Some(c) = iter.next() { if c == '\\' { match iter.next() { - Some('`') => s.push('`'), - Some(c) => { s.push('\\'); s.push(c); } - None => s.push('\\'), + Some('`') => line.push('`'), + Some(c) => { line.push('\\'); line.push(c); } + None => line.push('\\'), } } else if is_newline_char(c) { if c == '\r' && iter.peek() == Some(&'\n') { iter.next(); } - lines.push(std::mem::replace(&mut s, String::new())); + lines.push(std::mem::take(&mut line)); } else { - s.push(c); + line.push(c); } } - lines.push(s); + lines.push(line); lines } - #[cfg(test)] #[allow(non_snake_case)] mod tests { use crate::size::Size; - use crate::syntax::test::{DebugFn, check}; - use crate::syntax::func::Value; + use super::super::test::{check, DebugFn}; + use super::super::func::Value; use super::*; use Decoration::*; @@ -666,8 +632,8 @@ mod tests { scope.add::("box"); scope.add::("val"); - let ctx = ParseContext { scope: &scope }; - let pass = parse(Position::ZERO, $source, ctx); + let state = ParseState { scope }; + let pass = parse($source, Position::ZERO, &state); // Test model. let (exp, cmp) = span_vec![$($model)*]; @@ -839,14 +805,14 @@ mod tests { fn parse_function_names() { // No closing bracket. p!("[" => [func!("")], [ - (0:1, 0:1, "expected identifier"), + (0:1, 0:1, "expected function name"), (0:1, 0:1, "expected closing bracket") ]); // No name. - p!("[]" => [func!("")], [(0:1, 0:1, "expected identifier")]); + p!("[]" => [func!("")], [(0:1, 0:1, "expected function name")]); p!("[\"]" => [func!("")], [ - (0:1, 0:3, "expected identifier, found string"), + (0:1, 0:3, "expected function name, found string"), (0:3, 0:3, "expected closing bracket"), ]); @@ -862,9 +828,9 @@ mod tests { p!("[ f]" => [func!("f")], [], [(0:3, 0:4, ValidFuncName)]); // An invalid token for a name. - p!("[12]" => [func!("")], [(0:1, 0:3, "expected identifier, found number")], []); - p!("[🌎]" => [func!("")], [(0:1, 0:2, "expected identifier, found invalid token")], []); - p!("[ 🌎]" => [func!("")], [(0:3, 0:4, "expected identifier, found invalid token")], []); + p!("[12]" => [func!("")], [(0:1, 0:3, "expected function name, found number")], []); + p!("[🌎]" => [func!("")], [(0:1, 0:2, "expected function name, found invalid token")], []); + p!("[ 🌎]" => [func!("")], [(0:3, 0:4, "expected function name, found invalid token")], []); } #[test] @@ -954,6 +920,11 @@ mod tests { Num(2.0) ))); + // Associativity of multiplication and division. + p!("[val: 3/4*5]" => + [func!("val": (Mul(Div(Num(3.0), Num(4.0)), Num(5.0))), {})] + ); + // Invalid expressions. p!("[val: 4pt--]" => [func!("val": (Pt(4.0)))], [ (0:10, 0:11, "dangling minus"), @@ -971,6 +942,12 @@ mod tests { pval!("()" => (tuple!())); pval!("empty()" => (named_tuple!("empty"))); + // Space between name and tuple. + pval!("add ( 1 , 2 )" => (named_tuple!("add", Num(1.0), Num(2.0)))); + pval!("num = add ( 1 , 2 )" => (), { + "num" => named_tuple!("add", Num(1.0), Num(2.0)) + }); + // Invalid value. p!("[val: sound(\x07)]" => [func!("val": (named_tuple!("sound")), {})], diff --git a/src/syntax/scope.rs b/src/syntax/scope.rs index f7b20b9f3..74c64280b 100644 --- a/src/syntax/scope.rs +++ b/src/syntax/scope.rs @@ -5,12 +5,10 @@ use std::fmt::{self, Debug, Formatter}; use crate::Pass; use crate::func::ParseFunc; -use super::func::FuncHeader; -use super::parsing::ParseContext; -use super::span::Spanned; +use super::func::FuncCall; +use super::parsing::ParseState; use super::Model; - /// A map from identifiers to function parsers. pub struct Scope { parsers: HashMap>, @@ -50,10 +48,8 @@ impl Scope { } /// Return the parser with the given name if there is one. - pub fn get_parser(&self, name: &str) -> Result<&Parser, &Parser> { - self.parsers.get(name) - .map(|x| &**x) - .ok_or_else(|| &*self.fallback) + pub fn get_parser(&self, name: &str) -> Option<&Parser> { + self.parsers.get(name).map(AsRef::as_ref) } /// Return the fallback parser. @@ -72,16 +68,12 @@ impl Debug for Scope { /// A function which parses the source of a function into a model type which /// implements [`Model`]. -type Parser = dyn Fn( - FuncHeader, - Option>, - ParseContext, -) -> Pass>; +type Parser = dyn Fn(FuncCall, &ParseState) -> Pass>; fn parser(metadata: ::Meta) -> Box where F: ParseFunc + Model + 'static { - Box::new(move |h, b, c| { - F::parse(h, b, c, metadata.clone()) + Box::new(move |f, s| { + F::parse(f, s, metadata.clone()) .map(|model| Box::new(model) as Box) }) } diff --git a/src/syntax/span.rs b/src/syntax/span.rs index 8083bc064..c8e2cddb2 100644 --- a/src/syntax/span.rs +++ b/src/syntax/span.rs @@ -4,14 +4,14 @@ use std::fmt::{self, Debug, Formatter}; use std::ops::{Add, Sub}; use serde::Serialize; -/// A vector of spanned things. +/// A vector of spanned values of type `T`. pub type SpanVec = Vec>; /// [Offset](Span::offset) all spans in a vector of spanned things by a start /// position. pub fn offset_spans( - start: Position, vec: SpanVec, + start: Position, ) -> impl Iterator> { vec.into_iter().map(move |s| s.map_span(|span| span.offset(start))) } @@ -95,6 +95,11 @@ impl Span { Span { start: pos, end: pos } } + /// Expand a span by merging it with another span. + pub fn expand(&mut self, other: Span) { + *self = Span::merge(*self, other) + } + /// Offset a span by a start position. /// /// This is, for example, used to translate error spans from function local @@ -126,7 +131,7 @@ impl Position { /// The line 0, column 0 position. pub const ZERO: Position = Position { line: 0, column: 0 }; - /// Crete a new instance from line and column. + /// Create a new position from line and column. pub fn new(line: usize, column: usize) -> Position { Position { line, column } } diff --git a/src/syntax/test.rs b/src/syntax/test.rs index aaad5c127..7b1e08304 100644 --- a/src/syntax/test.rs +++ b/src/syntax/test.rs @@ -63,13 +63,13 @@ function! { pub body: Option, } - parse(header, body, ctx, f) { + parse(header, body, state, f) { let cloned = header.clone(); header.args.pos.items.clear(); header.args.key.pairs.clear(); DebugFn { header: cloned, - body: body!(opt: body, ctx, f), + body: body!(opt: body, state, f), } } diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs index b1bf76e3e..102007080 100644 --- a/src/syntax/tokens.rs +++ b/src/syntax/tokens.rs @@ -5,9 +5,8 @@ use unicode_xid::UnicodeXID; use crate::size::Size; use super::span::{Position, Span, Spanned}; -use self::Token::*; -use self::TokenizationMode::*; - +use Token::*; +use TokenMode::*; /// A minimal semantic entity of source code. #[derive(Debug, Copy, Clone, PartialEq)] @@ -152,7 +151,7 @@ impl<'s> Token<'s> { #[derive(Debug)] pub struct Tokens<'s> { src: &'s str, - mode: TokenizationMode, + mode: TokenMode, iter: Peekable>, position: Position, index: usize, @@ -163,20 +162,22 @@ pub struct Tokens<'s> { /// backtick tokens. #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)] #[allow(missing_docs)] -pub enum TokenizationMode { +pub enum TokenMode { Header, Body, } impl<'s> Tokens<'s> { - /// Create a new token iterator with the given mode where the first token - /// span starts an the given `start` position. - pub fn new(start: Position, src: &'s str, mode: TokenizationMode) -> Tokens<'s> { + /// Create a new token iterator with the given mode. + /// + /// The first token's span starts an the given `offset` position instead of + /// the zero position. + pub fn new(src: &'s str, offset: Position, mode: TokenMode) -> Tokens<'s> { Tokens { src, mode, iter: src.chars().peekable(), - position: start, + position: offset, index: 0, } } @@ -341,7 +342,7 @@ impl<'s> Tokens<'s> { Function { header, body: Some(Spanned { v: body, span }), terminated } } - fn read_function_part(&mut self, mode: TokenizationMode) -> (&'s str, bool) { + fn read_function_part(&mut self, mode: TokenMode) -> (&'s str, bool) { let start = self.index(); let mut terminated = false; @@ -530,7 +531,6 @@ pub fn is_identifier(string: &str) -> bool { true } - #[cfg(test)] #[allow(non_snake_case)] mod tests { @@ -557,7 +557,7 @@ mod tests { macro_rules! t { ($mode:expr, $source:expr => [$($tokens:tt)*]) => { let (exp, spans) = span_vec![$($tokens)*]; - let found = Tokens::new(Position::ZERO, $source, $mode).collect::>(); + let found = Tokens::new($source, Position::ZERO, $mode).collect::>(); check($source, exp, found, spans); } } From 9672d4320052d08b67d497febed4a0ad78bf9252 Mon Sep 17 00:00:00 2001 From: Laurenz Date: Wed, 29 Jul 2020 17:38:14 +0200 Subject: [PATCH 4/4] =?UTF-8?q?Improve=20argument=20naming=20and=20fix=20g?= =?UTF-8?q?rammar=20in=20comment=20=E2=99=BB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/lib.rs | 8 ++++---- src/syntax/parsing.rs | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 7f1752796..8f5bbdd61 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -204,10 +204,10 @@ impl Feedback { } /// Add more feedback whose spans are local and need to be offset by an - /// `offset` to be correct for this feedbacks context. - pub fn extend_offset(&mut self, other: Feedback, offset: Position) { - self.problems.extend(offset_spans(other.problems, offset)); - self.decos.extend(offset_spans(other.decos, offset)); + /// `offset` to be correct in this feedback's context. + pub fn extend_offset(&mut self, more: Feedback, offset: Position) { + self.problems.extend(offset_spans(more.problems, offset)); + self.decos.extend(offset_spans(more.decos, offset)); } } diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs index ece09a866..a0d9c4e45 100644 --- a/src/syntax/parsing.rs +++ b/src/syntax/parsing.rs @@ -30,7 +30,7 @@ pub fn parse(src: &str, offset: Position, state: &ParseState) -> Pass if newlines >= 2 { Node::Parbreak } else {