From bb1350cff5f1cf4da0332b72ec0703689645649c Mon Sep 17 00:00:00 2001 From: Martin Haug Date: Sat, 18 Jul 2020 14:04:58 +0200 Subject: [PATCH 1/4] Parsing mathematical expressions :heavy_plus_sign: --- src/syntax/expr.rs | 28 ++++-- src/syntax/parsing.rs | 203 ++++++++++++++++++++++++++++++++++++++---- src/syntax/test.rs | 31 +++++-- src/syntax/tokens.rs | 51 +++++++++-- 4 files changed, 273 insertions(+), 40 deletions(-) diff --git a/src/syntax/expr.rs b/src/syntax/expr.rs index 9d6902664..c340d9b0b 100644 --- a/src/syntax/expr.rs +++ b/src/syntax/expr.rs @@ -1,6 +1,6 @@ //! Expressions in function headers. -use std::fmt::{self, Write, Debug, Formatter}; +use std::fmt::{self, Debug, Formatter}; use std::iter::FromIterator; use std::ops::Deref; use std::str::FromStr; @@ -26,7 +26,7 @@ pub enum Expr { Size(Size), /// A bool: `true, false`. Bool(bool), - /// A color value, including the alpha channel: `#f79143ff` + /// A color value, including the alpha channel: `#f79143ff`. Color(RgbaColor), /// A tuple: `(false, 12cm, "hi")`. Tuple(Tuple), @@ -34,6 +34,16 @@ pub enum Expr { NamedTuple(NamedTuple), /// An object: `{ fit: false, size: 12pt }`. Object(Object), + /// An operator that negates the contained expression. + Neg(Box>), + /// An operator that adds the contained expressions. + Add(Box>, Box>), + /// An operator that subtracts contained expressions. + Sub(Box>, Box>), + /// An operator that multiplies the contained expressions. + Mul(Box>, Box>), + /// An operator that divides the contained expressions. + Div(Box>, Box>), } impl Expr { @@ -50,6 +60,11 @@ impl Expr { Tuple(_) => "tuple", NamedTuple(_) => "named tuple", Object(_) => "object", + Neg(_) => "negation", + Add(_, _) => "addition", + Sub(_, _) => "subtraction", + Mul(_, _) => "multiplication", + Div(_, _) => "division", } } } @@ -67,6 +82,11 @@ impl Debug for Expr { Tuple(t) => t.fmt(f), NamedTuple(t) => t.fmt(f), Object(o) => o.fmt(f), + Neg(e) => write!(f, "-{:?}", e), + Add(a, b) => write!(f, "({:?} + {:?})", a, b), + Sub(a, b) => write!(f, "({:?} - {:?})", a, b), + Mul(a, b) => write!(f, "({:?} * {:?})", a, b), + Div(a, b) => write!(f, "({:?} / {:?})", a, b), } } } @@ -102,9 +122,7 @@ impl Ident { impl Debug for Ident { fn fmt(&self, f: &mut Formatter) -> fmt::Result { - f.write_char('`')?; - f.write_str(&self.0)?; - f.write_char('`') + write!(f, "`{}`", self.0) } } diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs index cfbf2bf3f..1cb32263c 100644 --- a/src/syntax/parsing.rs +++ b/src/syntax/parsing.rs @@ -223,9 +223,109 @@ impl<'s> FuncParser<'s> { } }).v } - - /// Parse an atomic or compound (tuple / object) expression. + + /// Parse an expression which may contain math operands. + /// For this, this method looks for operators in + /// descending order of associativity, i.e. we first drill down to find all + /// negations, brackets and tuples, the next level, + /// we look for multiplication and division and here finally, for addition + /// and subtraction. fn parse_expr(&mut self) -> Option> { + let term = self.parse_term()?; + self.skip_whitespace(); + + if let Some(next) = self.peek() { + match next.v { + Token::Plus | Token::Hyphen => { + self.eat(); + self.skip_whitespace(); + let o2 = self.parse_expr(); + if o2.is_none() { + self.feedback.errors.push(err!( + Span::merge(next.span, term.span); + "Missing right summand" + )); + return Some(term) + } + + let o2 = o2.expect("Checked for None before"); + let span = Span::merge(term.span, o2.span); + match next.v { + Token::Plus => Some(Spanned::new( + Expr::Add(Box::new(term), Box::new(o2)), span + )), + Token::Hyphen => Some(Spanned::new( + Expr::Sub(Box::new(term), Box::new(o2)), span + )), + _ => unreachable!(), + } + }, + _ => Some(term) + } + } else { + Some(term) + } + } + + fn parse_term(&mut self) -> Option> { + // TODO: Deduplicate code here + let factor = self.parse_factor()?; + self.skip_whitespace(); + + if let Some(next) = self.peek() { + match next.v { + Token::Star | Token::Slash => { + self.eat(); + self.skip_whitespace(); + let o2 = self.parse_term(); + if o2.is_none() { + self.feedback.errors.push(err!( + Span::merge(next.span, factor.span); + "Missing right factor" + )); + return Some(factor) + } + + let o2 = o2.expect("Checked for None before"); + let span = Span::merge(factor.span, o2.span); + match next.v { + Token::Star => Some(Spanned::new( + Expr::Mul(Box::new(factor), Box::new(o2)), span + )), + Token::Slash => Some(Spanned::new( + Expr::Div(Box::new(factor), Box::new(o2)), span + )), + _ => unreachable!(), + } + }, + _ => Some(factor) + } + } else { + Some(factor) + } + } + + /// Parse expressions that are of the form value or -value + fn parse_factor(&mut self) -> Option> { + let first = self.peek()?; + if first.v == Token::Hyphen { + self.eat(); + let o2 = self.parse_value(); + self.skip_whitespace(); + if o2.is_none() { + self.feedback.errors.push(err!(first.span; "Dangling minus")); + return None + } + + let o2 = o2.expect("Checked for None before"); + let span = Span::merge(first.span, o2.span); + Some(Spanned::new(Expr::Neg(Box::new(o2)), span)) + } else { + self.parse_value() + } + } + + fn parse_value(&mut self) -> Option> { let first = self.peek()?; macro_rules! take { ($v:expr) => ({ self.eat(); Spanned { v: $v, span: first.span } }); @@ -263,27 +363,36 @@ impl<'s> FuncParser<'s> { } }, - Token::LeftParen => self.parse_tuple().map(|t| Expr::Tuple(t)), + Token::LeftParen => { + let mut tuple = self.parse_tuple(); + // Coerce 1-tuple into value + if tuple.1 && tuple.0.v.items.len() > 0 { + tuple.0.v.items.pop().expect("Length is one") + } else { + tuple.0.map(|t| Expr::Tuple(t)) + } + }, Token::LeftBrace => self.parse_object().map(|o| Expr::Object(o)), _ => return None, }) } - /// Parse a tuple expression: `(, ...)`. - fn parse_tuple(&mut self) -> Spanned { + /// Parse a tuple expression: `(, ...)`. The boolean in the return + /// values showes whether the tuple can be coerced into a single value + fn parse_tuple(&mut self) -> (Spanned, bool) { let token = self.eat(); debug_assert_eq!(token.map(Spanned::value), Some(Token::LeftParen)); // Parse a collection until a right paren appears and complain about // missing a `value` when an invalid token is encoutered. - self.parse_collection(Some(Token::RightParen), + self.parse_collection_bracket_aware(Some(Token::RightParen), |p| p.parse_expr().ok_or(("value", None))) } /// Parse a tuple expression: `name(, ...)` with a given identifier. fn parse_named_tuple(&mut self, name: Spanned) -> Spanned { - let tuple = self.parse_tuple(); + let tuple = self.parse_tuple().0; let span = Span::merge(name.span, tuple.span); Spanned::new(NamedTuple::new(name, tuple), span) } @@ -319,17 +428,21 @@ impl<'s> FuncParser<'s> { } /// Parse a comma-separated collection where each item is parsed through - /// `parse_item` until the `end` token is met. - fn parse_collection( + /// `parse_item` until the `end` token is met. The first item in the return + /// tuple is the collection, the second item indicates whether the + /// collection can be coerced into a single item + /// (i.e. at least one comma appeared). + fn parse_collection_bracket_aware( &mut self, end: Option, mut parse_item: F - ) -> Spanned + ) -> (Spanned, bool) where C: FromIterator, F: FnMut(&mut Self) -> Result)>, { let start = self.pos(); + let mut can_be_coerced = true; // Parse the comma separated items. let collection = std::iter::from_fn(|| { @@ -359,8 +472,14 @@ impl<'s> FuncParser<'s> { let behind_item = self.pos(); self.skip_whitespace(); match self.peekv() { - Some(Token::Comma) => { self.eat(); } - t @ Some(_) if t != end => self.expected_at("comma", behind_item), + Some(Token::Comma) => { + can_be_coerced = false; + self.eat(); + } + t @ Some(_) if t != end => { + can_be_coerced = false; + self.expected_at("comma", behind_item); + }, _ => {} } @@ -383,7 +502,21 @@ impl<'s> FuncParser<'s> { }).filter_map(|x| x).collect(); let end = self.pos(); - Spanned::new(collection, Span { start, end }) + (Spanned::new(collection, Span { start, end }), can_be_coerced) + } + + /// Parse a comma-separated collection where each item is parsed through + /// `parse_item` until the `end` token is met. + fn parse_collection( + &mut self, + end: Option, + parse_item: F + ) -> Spanned + where + C: FromIterator, + F: FnMut(&mut Self) -> Result)>, + { + self.parse_collection_bracket_aware(end, parse_item).0 } /// Try to parse an identifier and do nothing if the peekable token is no @@ -546,6 +679,19 @@ mod tests { fn Id(text: &str) -> Expr { Expr::Ident(Ident(text.to_string())) } fn Str(text: &str) -> Expr { Expr::Str(text.to_string()) } fn Pt(points: f32) -> Expr { Expr::Size(Size::pt(points)) } + fn Neg(e1: Expr) -> Expr { Expr::Neg(Box::new(zspan(e1))) } + fn Add(e1: Expr, e2: Expr) -> Expr { + Expr::Add(Box::new(zspan(e1)), Box::new(zspan(e2))) + } + fn Sub(e1: Expr, e2: Expr) -> Expr { + Expr::Sub(Box::new(zspan(e1)), Box::new(zspan(e2))) + } + fn Mul(e1: Expr, e2: Expr) -> Expr { + Expr::Mul(Box::new(zspan(e1)), Box::new(zspan(e2))) + } + fn Div(e1: Expr, e2: Expr) -> Expr { + Expr::Div(Box::new(zspan(e1)), Box::new(zspan(e2))) + } fn Clr(r: u8, g: u8, b: u8, a: u8) -> Expr { Expr::Color(RgbaColor::new(r, g, b, a)) @@ -814,6 +960,12 @@ mod tests { p!("[val: 12e1pt]" => [func!("val": (Pt(12e1)), {})]); p!("[val: #f7a20500]" => [func!("val": (ClrStr("f7a20500")), {})]); + // Math + p!("[val: 3.2in + 6pt]" => [func!("val": (Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0)))), {})]); + p!("[val: 5 - 0.01]" => [func!("val": (Sub(Num(5.0), Num(0.01))), {})]); + p!("[val: (3mm * 2)]" => [func!("val": (Mul(Sz(Size::mm(3.0)), Num(2.0))), {})]); + p!("[val: 12e-3cm/1pt]" => [func!("val": (Div(Sz(Size::cm(12e-3)), Sz(Size::pt(1.0)))), {})]); + // Unclosed string. p!("[val: \"hello]" => [func!("val": (Str("hello]")), {})], [ (0:13, 0:13, "expected quote"), @@ -835,6 +987,19 @@ mod tests { ]); } + #[test] + fn parse_complex_mathematical_expressions() { + p!("[val: (3.2in + 6pt)*(5/2-1)]" => [func!("val": ( + Mul( + Add(Sz(Size::inches(3.2)), Sz(Size::pt(6.0))), + Sub(Div(Num(5.0), Num(2.0)), Num(1.0)) + ) + ), {})]); + p!("[val: (6.3E+2+4*-3.2pt)/2]" => [func!("val": ( + Div(Add(Num(6.3e2),Mul(Num(4.0), Neg(Pt(3.2)))), Num(2.0)) + ), {})]); + } + #[test] fn parse_tuples() { // Empty tuple @@ -858,9 +1023,9 @@ mod tests { ); // Unclosed tuple - p!("[val: (hello]" => - [func!("val": (tuple!(Id("hello"))), {})], - [(0:12, 0:12, "expected closing paren")], + p!("[val: (hello,]" => + [func!("val": (tuple!(Id("hello"),)), {})], + [(0:13, 0:13, "expected closing paren")], ); p!("[val: lang(中文]" => [func!("val": (named_tuple!("lang", Id("中文"))), {})], @@ -882,8 +1047,8 @@ mod tests { ); // Nested tuples - p!("[val: (1, (2))]" => - [func!("val": (tuple!(Num(1.0), tuple!(Num(2.0)))), {})] + p!("[val: (1, (2, 3))]" => + [func!("val": (tuple!(Num(1.0), tuple!(Num(2.0), Num(3.0)))), {})] ); p!("[val: css(1pt, rgb(90, 102, 254), \"solid\")]" => [func!("val": (named_tuple!( @@ -1085,7 +1250,7 @@ mod tests { // Body nodes in bodies. p!("[val:*][*Hi*]" => [func!("val"; [Bold, T("Hi"), Bold])], - [(0:5, 0:6, "expected argument, found invalid token")], + [(0:5, 0:6, "expected argument, found star")], ); // Errors in bodies. diff --git a/src/syntax/test.rs b/src/syntax/test.rs index ca47c0c55..109b5ef6d 100644 --- a/src/syntax/test.rs +++ b/src/syntax/test.rs @@ -88,13 +88,6 @@ pub trait SpanlessEq { fn spanless_eq(&self, other: &Rhs) -> bool; } -impl SpanlessEq for Vec> { - fn spanless_eq(&self, other: &Vec>) -> bool { - self.len() == other.len() - && self.iter().zip(other).all(|(x, y)| x.v.spanless_eq(&y.v)) - } -} - impl SpanlessEq for SyntaxModel { fn spanless_eq(&self, other: &SyntaxModel) -> bool { self.nodes.spanless_eq(&other.nodes) @@ -130,6 +123,11 @@ impl SpanlessEq for Expr { (Expr::NamedTuple(a), Expr::NamedTuple(b)) => a.spanless_eq(b), (Expr::Tuple(a), Expr::Tuple(b)) => a.spanless_eq(b), (Expr::Object(a), Expr::Object(b)) => a.spanless_eq(b), + (Expr::Neg(a), Expr::Neg(b)) => a.spanless_eq(&b), + (Expr::Add(a1, a2), Expr::Add(b1, b2)) => a1.spanless_eq(&b1) && a2.spanless_eq(&b2), + (Expr::Sub(a1, a2), Expr::Sub(b1, b2)) => a1.spanless_eq(&b1) && a2.spanless_eq(&b2), + (Expr::Mul(a1, a2), Expr::Mul(b1, b2)) => a1.spanless_eq(&b1) && a2.spanless_eq(&b2), + (Expr::Div(a1, a2), Expr::Div(b1, b2)) => a1.spanless_eq(&b1) && a2.spanless_eq(&b2), (a, b) => a == b, } } @@ -158,6 +156,25 @@ impl SpanlessEq for Object { } } +impl SpanlessEq for Vec { + fn spanless_eq(&self, other: &Vec) -> bool { + self.len() == other.len() + && self.iter().zip(other).all(|(x, y)| x.spanless_eq(&y)) + } +} + +impl SpanlessEq for Spanned { + fn spanless_eq(&self, other: &Spanned) -> bool { + self.v.spanless_eq(&other.v) + } +} + +impl SpanlessEq for Box { + fn spanless_eq(&self, other: &Box) -> bool { + (&**self).spanless_eq(&**other) + } +} + /// Implement `SpanlessEq` by just forwarding to `PartialEq`. macro_rules! forward { ($type:ty) => { diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs index adf29de23..6642a9bfa 100644 --- a/src/syntax/tokens.rs +++ b/src/syntax/tokens.rs @@ -78,10 +78,18 @@ pub enum Token<'s> { ExprSize(Size), /// A boolean in a function header: `true | false`. ExprBool(bool), - /// A hex value in a function header: `#20d82a` + /// A hex value in a function header: `#20d82a`. ExprHex(&'s str), + /// A plus in a function header, signifying the addition of expressions. + Plus, + /// A hyphen in a function header, + /// signifying the subtraction of expressions. + Hyphen, + /// A slash in a function header, signifying the division of expressions. + Slash, - /// A star in body-text. + /// A star. It can appear in a function header where it signifies the + /// multiplication of expressions or the body where it modifies the styling. Star, /// An underscore in body-text. Underscore, @@ -125,6 +133,9 @@ impl<'s> Token<'s> { ExprSize(_) => "size", ExprBool(_) => "bool", ExprHex(_) => "hex value", + Plus => "plus", + Hyphen => "minus", + Slash => "slash", Star => "star", Underscore => "underscore", Backslash => "backslash", @@ -212,12 +223,20 @@ impl<'s> Iterator for Tokens<'s> { ':' if self.mode == Header => Colon, ',' if self.mode == Header => Comma, '=' if self.mode == Header => Equals, + + // Expression operators. + '+' if self.mode == Header => Plus, + '-' if self.mode == Header => Hyphen, + '/' if self.mode == Header => Slash, // String values. '"' if self.mode == Header => self.parse_string(), + // Star serves a double purpose as a style modifier + // and a expression operator in the header. + '*' => Star, + // Style toggles. - '*' if self.mode == Body => Star, '_' if self.mode == Body => Underscore, '`' if self.mode == Body => self.parse_raw(), @@ -231,15 +250,20 @@ impl<'s> Iterator for Tokens<'s> { c => { let body = self.mode == Body; + let mut last_was_e = false; let text = self.read_string_until(|n| { - match n { + let val = match n { c if c.is_whitespace() => true, - '[' | ']' | '/' => true, - '\\' | '*' | '_' | '`' if body => true, + '[' | ']' | '/' | '*' => true, + '\\' | '_' | '`' if body => true, ':' | '=' | ',' | '"' | '(' | ')' | '{' | '}' if !body => true, + '+' | '-' if !body && !last_was_e => true, _ => false, - } + }; + + last_was_e = n == 'e' || n == 'E'; + val }, false, -(c.len_utf8() as isize), 0).0; if self.mode == Header { @@ -411,6 +435,8 @@ impl<'s> Tokens<'s> { } } + /// Will read the input stream until the argument F evaluates to `true` + /// for the current character. fn read_string_until( &mut self, mut f: F, @@ -517,6 +543,10 @@ mod tests { ExprBool as Bool, ExprHex as Hex, Text as T, + Plus, + Hyphen as Min, + Star, + Slash, }; #[allow(non_snake_case)] @@ -595,7 +625,7 @@ mod tests { t!(Body, "`]" => [Raw("]", false)]); t!(Body, "`\\``" => [Raw("\\`", true)]); t!(Body, "\\ " => [Backslash, S(0)]); - t!(Header, "_*`" => [Invalid("_*`")]); + t!(Header, "_`" => [Invalid("_`")]); } #[test] @@ -613,10 +643,13 @@ mod tests { t!(Header, "12e4%" => [Num(1200.0)]); t!(Header, "__main__" => [Id("__main__")]); t!(Header, ".func.box" => [Id(".func.box")]); - t!(Header, "--arg, _b, _1" => [Id("--arg"), Comma, S(0), Id("_b"), Comma, S(0), Id("_1")]); + t!(Header, "arg, _b, _1" => [Id("arg"), Comma, S(0), Id("_b"), Comma, S(0), Id("_1")]); t!(Header, "12_pt, 12pt" => [Invalid("12_pt"), Comma, S(0), Sz(Size::pt(12.0))]); t!(Header, "1e5in" => [Sz(Size::inches(100000.0))]); t!(Header, "2.3cm" => [Sz(Size::cm(2.3))]); + t!(Header, "12e-3in" => [Sz(Size::inches(12e-3))]); + t!(Header, "6.1cm + 4pt,a=1*2" => [Sz(Size::cm(6.1)), S(0), Plus, S(0), Sz(Size::pt(4.0)), Comma, Id("a"), Equals, Num(1.0), Star, Num(2.0)]); + t!(Header, "(5 - 1) / 2.1" => [LP, Num(5.0), S(0), Min, S(0), Num(1.0), RP, S(0), Slash, S(0), Num(2.1)]); t!(Header, "02.4mm" => [Sz(Size::mm(2.4))]); t!(Header, "2.4.cm" => [Invalid("2.4.cm")]); t!(Header, "(1,2)" => [LP, Num(1.0), Comma, Num(2.0), RP]); From e9a9581252080853418d386992cdadecbce9f7dc Mon Sep 17 00:00:00 2001 From: Martin Haug Date: Sat, 18 Jul 2020 16:57:58 +0200 Subject: [PATCH 2/4] =?UTF-8?q?Spanned=20object=20pairs=20and=20refactorin?= =?UTF-8?q?g=20=F0=9F=A7=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/syntax/expr.rs | 28 +++---- src/syntax/func/mod.rs | 10 +-- src/syntax/parsing.rs | 172 ++++++++++++++++++++--------------------- src/syntax/test.rs | 2 +- 4 files changed, 104 insertions(+), 108 deletions(-) diff --git a/src/syntax/expr.rs b/src/syntax/expr.rs index c340d9b0b..1dc47d03a 100644 --- a/src/syntax/expr.rs +++ b/src/syntax/expr.rs @@ -364,7 +364,7 @@ impl Deref for NamedTuple { #[derive(Default, Clone, PartialEq)] pub struct Object { /// The key-value pairs of the object. - pub pairs: Vec, + pub pairs: Vec>, } /// A key-value pair in an object. @@ -391,7 +391,7 @@ impl Object { } /// Add a pair to object. - pub fn add(&mut self, pair: Pair) { + pub fn add(&mut self, pair: Spanned) { self.pairs.push(pair); } @@ -401,7 +401,7 @@ impl Object { /// Inserts an error if the value does not match. If the key is not /// contained, no error is inserted. pub fn get(&mut self, errors: &mut Errors, key: &str) -> Option { - let index = self.pairs.iter().position(|pair| pair.key.v.as_str() == key)?; + let index = self.pairs.iter().position(|pair| pair.v.key.v.as_str() == key)?; self.get_index::(errors, index) } @@ -414,7 +414,7 @@ impl Object { errors: &mut Errors, ) -> Option<(K, V)> { for (index, pair) in self.pairs.iter().enumerate() { - let key = Spanned { v: pair.key.v.as_str(), span: pair.key.span }; + let key = Spanned { v: pair.v.key.v.as_str(), span: pair.v.key.span }; if let Some(key) = K::parse(key) { return self.get_index::(errors, index).map(|value| (key, value)); } @@ -432,7 +432,7 @@ impl Object { let mut index = 0; std::iter::from_fn(move || { if index < self.pairs.len() { - let key = &self.pairs[index].key; + let key = &self.pairs[index].v.key; let key = Spanned { v: key.v.as_str(), span: key.span }; Some(if let Some(key) = K::parse(key) { @@ -465,7 +465,7 @@ impl Object { /// Extract the argument at the given index and insert an error if the value /// does not match. fn get_index(&mut self, errors: &mut Errors, index: usize) -> Option { - let expr = self.pairs.remove(index).value; + let expr = self.pairs.remove(index).v.value; let span = expr.span; match V::parse(expr) { Ok(output) => Some(output), @@ -474,14 +474,14 @@ impl Object { } /// Iterate over the pairs of this object. - pub fn iter<'a>(&'a self) -> std::slice::Iter<'a, Pair> { + pub fn iter<'a>(&'a self) -> std::slice::Iter<'a, Spanned> { self.pairs.iter() } } impl IntoIterator for Object { - type Item = Pair; - type IntoIter = std::vec::IntoIter; + type Item = Spanned; + type IntoIter = std::vec::IntoIter>; fn into_iter(self) -> Self::IntoIter { self.pairs.into_iter() @@ -489,16 +489,16 @@ impl IntoIterator for Object { } impl<'a> IntoIterator for &'a Object { - type Item = &'a Pair; - type IntoIter = std::slice::Iter<'a, Pair>; + type Item = &'a Spanned; + type IntoIter = std::slice::Iter<'a, Spanned>; fn into_iter(self) -> Self::IntoIter { self.iter() } } -impl FromIterator for Object { - fn from_iter>(iter: I) -> Self { +impl FromIterator> for Object { + fn from_iter>>(iter: I) -> Self { Object { pairs: iter.into_iter().collect() } } } @@ -506,7 +506,7 @@ impl FromIterator for Object { impl Debug for Object { fn fmt(&self, f: &mut Formatter) -> fmt::Result { f.debug_map() - .entries(self.pairs.iter().map(|p| (&p.key.v, &p.value.v))) + .entries(self.pairs.iter().map(|p| (&p.v.key.v, &p.v.value.v))) .finish() } } diff --git a/src/syntax/func/mod.rs b/src/syntax/func/mod.rs index 53bbc14ef..54d34531c 100644 --- a/src/syntax/func/mod.rs +++ b/src/syntax/func/mod.rs @@ -56,11 +56,11 @@ impl FuncArgs { } } -impl FromIterator for FuncArgs { - fn from_iter>(iter: I) -> Self { +impl FromIterator> for FuncArgs { + fn from_iter>>(iter: I) -> Self { let mut args = FuncArgs::new(); for item in iter.into_iter() { - args.add(item); + args.add(item.v); } args } @@ -72,7 +72,7 @@ pub enum FuncArg { /// A positional argument. Pos(Spanned), /// A keyword argument. - Key(Pair), + Key(Spanned), } impl FuncArg { @@ -83,7 +83,7 @@ impl FuncArg { pub fn span(&self) -> Span { match self { FuncArg::Pos(item) => item.span, - FuncArg::Key(Pair { key, value }) => Span::merge(key.span, value.span), + FuncArg::Key(item) => item.span, } } } diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs index 1cb32263c..8ebb3a29e 100644 --- a/src/syntax/parsing.rs +++ b/src/syntax/parsing.rs @@ -190,10 +190,10 @@ impl<'s> FuncParser<'s> { if let Some(ident) = p.parse_ident() { // This could still be a named tuple if let Some(Token::LeftParen) = p.peekv() { - return Ok(FuncArg::Pos( - p.parse_named_tuple(ident) - .map(|t| Expr::NamedTuple(t)) - )); + let n_tuple = p.parse_named_tuple(ident) + .map(|t| Expr::NamedTuple(t)); + let span = n_tuple.span; + return Ok(Spanned::new(FuncArg::Pos(n_tuple), span)); } p.skip_whitespace(); @@ -209,16 +209,22 @@ impl<'s> FuncParser<'s> { let value = p.parse_expr().ok_or(("value", None))?; // Add a keyword argument. - Ok(FuncArg::Key(Pair { key: ident, value })) + let span = Span::merge(ident.span, value.span); + Ok(Spanned::new( + FuncArg::Key(Spanned::new(Pair { key: ident, value }, span)), + span, + )) } else { // Add a positional argument because there was no equals // sign after the identifier that could have been a key. - Ok(FuncArg::Pos(ident.map(|id| Expr::Ident(id)))) + let ident = ident.map(|id| Expr::Ident(id)); + let span = ident.span; + Ok(Spanned::new(FuncArg::Pos(ident), span)) } } else { // Add a positional argument because we haven't got an // identifier that could be an argument key. - p.parse_expr().map(|expr| FuncArg::Pos(expr)) + p.parse_expr().map(|expr| Spanned { span: expr.span, v: FuncArg::Pos(expr) }) .ok_or(("argument", None)) } }).v @@ -231,78 +237,58 @@ impl<'s> FuncParser<'s> { /// we look for multiplication and division and here finally, for addition /// and subtraction. fn parse_expr(&mut self) -> Option> { - let term = self.parse_term()?; - self.skip_whitespace(); - - if let Some(next) = self.peek() { - match next.v { - Token::Plus | Token::Hyphen => { - self.eat(); - self.skip_whitespace(); - let o2 = self.parse_expr(); - if o2.is_none() { - self.feedback.errors.push(err!( - Span::merge(next.span, term.span); - "Missing right summand" - )); - return Some(term) - } - - let o2 = o2.expect("Checked for None before"); - let span = Span::merge(term.span, o2.span); - match next.v { - Token::Plus => Some(Spanned::new( - Expr::Add(Box::new(term), Box::new(o2)), span - )), - Token::Hyphen => Some(Spanned::new( - Expr::Sub(Box::new(term), Box::new(o2)), span - )), - _ => unreachable!(), - } - }, - _ => Some(term) - } - } else { - Some(term) - } + let o1 = self.parse_term()?; + self.parse_binop(o1, "summand", Self::parse_expr, |token| match token { + Token::Plus => Some(Expr::Add), + Token::Hyphen => Some(Expr::Sub), + _ => None, + }) } fn parse_term(&mut self) -> Option> { - // TODO: Deduplicate code here - let factor = self.parse_factor()?; + let o1 = self.parse_factor()?; + self.parse_binop(o1, "factor", Self::parse_term, |token| match token { + Token::Star => Some(Expr::Mul), + Token::Slash => Some(Expr::Div), + _ => None, + }) + } + + fn parse_binop( + &mut self, + o1: Spanned, + operand_name: &str, + mut parse_operand: F, + parse_op: G, + ) -> Option> + where + F: FnMut(&mut Self) -> Option>, + G: FnOnce(Token) -> Option>, Box>) -> Expr>, + { self.skip_whitespace(); if let Some(next) = self.peek() { - match next.v { - Token::Star | Token::Slash => { - self.eat(); - self.skip_whitespace(); - let o2 = self.parse_term(); - if o2.is_none() { - self.feedback.errors.push(err!( - Span::merge(next.span, factor.span); - "Missing right factor" - )); - return Some(factor) - } + let binop = match parse_op(next.v) { + Some(op) => op, + None => return Some(o1), + }; - let o2 = o2.expect("Checked for None before"); - let span = Span::merge(factor.span, o2.span); - match next.v { - Token::Star => Some(Spanned::new( - Expr::Mul(Box::new(factor), Box::new(o2)), span - )), - Token::Slash => Some(Spanned::new( - Expr::Div(Box::new(factor), Box::new(o2)), span - )), - _ => unreachable!(), - } - }, - _ => Some(factor) + self.eat(); + self.skip_whitespace(); + + if let Some(o2) = parse_operand(self) { + let span = Span::merge(o1.span, o2.span); + let expr = binop(Box::new(o1), Box::new(o2)); + return Some(Spanned::new(expr, span)); + } else { + self.feedback.errors.push(err!( + Span::merge(next.span, o1.span); + "missing right {}", operand_name, + )); } - } else { - Some(factor) } + + Some(o1) } /// Parse expressions that are of the form value or -value @@ -310,16 +296,15 @@ impl<'s> FuncParser<'s> { let first = self.peek()?; if first.v == Token::Hyphen { self.eat(); - let o2 = self.parse_value(); self.skip_whitespace(); - if o2.is_none() { - self.feedback.errors.push(err!(first.span; "Dangling minus")); - return None + + if let Some(factor) = self.parse_value() { + let span = Span::merge(first.span, factor.span); + Some(Spanned::new(Expr::Neg(Box::new(factor)), span)) + } else { + self.feedback.errors.push(err!(first.span; "dangling minus")); + None } - - let o2 = o2.expect("Checked for None before"); - let span = Span::merge(first.span, o2.span); - Some(Spanned::new(Expr::Neg(Box::new(o2)), span)) } else { self.parse_value() } @@ -423,7 +408,8 @@ impl<'s> FuncParser<'s> { let value = p.parse_expr().ok_or(("value", None))?; - Ok(Pair { key, value }) + let span = Span::merge(key.span, value.span); + Ok(Spanned::new(Pair { key, value }, span)) }) } @@ -438,8 +424,8 @@ impl<'s> FuncParser<'s> { mut parse_item: F ) -> (Spanned, bool) where - C: FromIterator, - F: FnMut(&mut Self) -> Result)>, + C: FromIterator>, + F: FnMut(&mut Self) -> Result, (&'static str, Option)>, { let start = self.pos(); let mut can_be_coerced = true; @@ -469,7 +455,6 @@ impl<'s> FuncParser<'s> { Ok(item) => { // Expect a comma behind the item (only separated by // whitespace). - let behind_item = self.pos(); self.skip_whitespace(); match self.peekv() { Some(Token::Comma) => { @@ -478,7 +463,7 @@ impl<'s> FuncParser<'s> { } t @ Some(_) if t != end => { can_be_coerced = false; - self.expected_at("comma", behind_item); + self.expected_at("comma", item.span.end); }, _ => {} } @@ -513,8 +498,8 @@ impl<'s> FuncParser<'s> { parse_item: F ) -> Spanned where - C: FromIterator, - F: FnMut(&mut Self) -> Result)>, + C: FromIterator>, + F: FnMut(&mut Self) -> Result, (&'static str, Option)>, { self.parse_collection_bracket_aware(end, parse_item).0 } @@ -735,10 +720,10 @@ mod tests { macro_rules! object { ($($key:expr => $value:expr),* $(,)?) => { Expr::Object(Object { - pairs: vec![$(Pair { + pairs: vec![$(zspan(Pair { key: zspan(Ident($key.to_string())), value: zspan($value), - }),*] + })),*] }) }; } @@ -995,9 +980,20 @@ mod tests { Sub(Div(Num(5.0), Num(2.0)), Num(1.0)) ) ), {})]); - p!("[val: (6.3E+2+4*-3.2pt)/2]" => [func!("val": ( + p!("[val: (6.3E+2+4* - 3.2pt)/2]" => [func!("val": ( Div(Add(Num(6.3e2),Mul(Num(4.0), Neg(Pt(3.2)))), Num(2.0)) ), {})]); + p!("[val: 4pt--]" => + [func!("val": (Pt(4.0)), {})], + [ + (0:10, 0:11, "dangling minus"), + (0:6, 0:10, "missing right summand") + ], + ); + p!("[val: 3mm+4pt*]" => + [func!("val": (Add(Sz(Size::mm(3.0)), Pt(4.0))), {})], + [(0:10, 0:14, "missing right factor")], + ); } #[test] diff --git a/src/syntax/test.rs b/src/syntax/test.rs index 109b5ef6d..47680487b 100644 --- a/src/syntax/test.rs +++ b/src/syntax/test.rs @@ -152,7 +152,7 @@ impl SpanlessEq for Object { fn spanless_eq(&self, other: &Object) -> bool { self.pairs.len() == other.pairs.len() && self.pairs.iter().zip(&other.pairs) - .all(|(x, y)| x.key.v == y.key.v && x.value.v.spanless_eq(&y.value.v)) + .all(|(x, y)| x.v.key.v == y.v.key.v && x.v.value.v.spanless_eq(&y.v.value.v)) } } From 332f83ed2dd5a050d1af145fc285003a75f39617 Mon Sep 17 00:00:00 2001 From: Laurenz Date: Sat, 18 Jul 2020 17:52:12 +0200 Subject: [PATCH 3/4] =?UTF-8?q?Some=20code=20and=20styling=20improvements?= =?UTF-8?q?=20=F0=9F=8E=A8?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/syntax/parsing.rs | 107 ++++++++++++++++++++---------------------- src/syntax/tokens.rs | 8 ++-- 2 files changed, 55 insertions(+), 60 deletions(-) diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs index 8ebb3a29e..07dfec7cb 100644 --- a/src/syntax/parsing.rs +++ b/src/syntax/parsing.rs @@ -213,7 +213,7 @@ impl<'s> FuncParser<'s> { Ok(Spanned::new( FuncArg::Key(Spanned::new(Pair { key: ident, value }, span)), span, - )) + )) } else { // Add a positional argument because there was no equals // sign after the identifier that could have been a key. @@ -229,13 +229,12 @@ impl<'s> FuncParser<'s> { } }).v } - - /// Parse an expression which may contain math operands. - /// For this, this method looks for operators in - /// descending order of associativity, i.e. we first drill down to find all - /// negations, brackets and tuples, the next level, - /// we look for multiplication and division and here finally, for addition - /// and subtraction. + + /// Parse an expression which may contain math operands. For this, this + /// method looks for operators in descending order of associativity, i.e. we + /// first drill down to find all negations, brackets and tuples, the next + /// level, we look for multiplication and division and here finally, for + /// addition and subtraction. fn parse_expr(&mut self) -> Option> { let o1 = self.parse_term()?; self.parse_binop(o1, "summand", Self::parse_expr, |token| match token { @@ -258,46 +257,43 @@ impl<'s> FuncParser<'s> { &mut self, o1: Spanned, operand_name: &str, - mut parse_operand: F, + parse_operand: F, parse_op: G, ) -> Option> where - F: FnMut(&mut Self) -> Option>, + F: FnOnce(&mut Self) -> Option>, G: FnOnce(Token) -> Option>, Box>) -> Expr>, { self.skip_whitespace(); if let Some(next) = self.peek() { - let binop = match parse_op(next.v) { - Some(op) => op, - None => return Some(o1), - }; + if let Some(binop) = parse_op(next.v) { + self.eat(); + self.skip_whitespace(); - self.eat(); - self.skip_whitespace(); - - if let Some(o2) = parse_operand(self) { - let span = Span::merge(o1.span, o2.span); - let expr = binop(Box::new(o1), Box::new(o2)); - return Some(Spanned::new(expr, span)); - } else { - self.feedback.errors.push(err!( - Span::merge(next.span, o1.span); - "missing right {}", operand_name, - )); + if let Some(o2) = parse_operand(self) { + let span = Span::merge(o1.span, o2.span); + let expr = binop(Box::new(o1), Box::new(o2)); + return Some(Spanned::new(expr, span)); + } else { + self.feedback.errors.push(err!( + Span::merge(next.span, o1.span); + "missing right {}", operand_name, + )); + } } } Some(o1) } - /// Parse expressions that are of the form value or -value + /// Parse expressions that are of the form value or -value. fn parse_factor(&mut self) -> Option> { let first = self.peek()?; if first.v == Token::Hyphen { self.eat(); self.skip_whitespace(); - + if let Some(factor) = self.parse_value() { let span = Span::merge(first.span, factor.span); Some(Spanned::new(Expr::Neg(Box::new(factor)), span)) @@ -349,12 +345,12 @@ impl<'s> FuncParser<'s> { }, Token::LeftParen => { - let mut tuple = self.parse_tuple(); + let (mut tuple, can_be_coerced) = self.parse_tuple(); // Coerce 1-tuple into value - if tuple.1 && tuple.0.v.items.len() > 0 { - tuple.0.v.items.pop().expect("Length is one") + if can_be_coerced && tuple.v.items.len() > 0 { + tuple.v.items.pop().expect("length is at least one") } else { - tuple.0.map(|t| Expr::Tuple(t)) + tuple.map(|t| Expr::Tuple(t)) } }, Token::LeftBrace => self.parse_object().map(|o| Expr::Object(o)), @@ -363,15 +359,15 @@ impl<'s> FuncParser<'s> { }) } - /// Parse a tuple expression: `(, ...)`. The boolean in the return - /// values showes whether the tuple can be coerced into a single value + /// Parse a tuple expression: `(, ...)`. The boolean in the return + /// values showes whether the tuple can be coerced into a single value. fn parse_tuple(&mut self) -> (Spanned, bool) { let token = self.eat(); debug_assert_eq!(token.map(Spanned::value), Some(Token::LeftParen)); // Parse a collection until a right paren appears and complain about // missing a `value` when an invalid token is encoutered. - self.parse_collection_bracket_aware(Some(Token::RightParen), + self.parse_collection_comma_aware(Some(Token::RightParen), |p| p.parse_expr().ok_or(("value", None))) } @@ -413,12 +409,25 @@ impl<'s> FuncParser<'s> { }) } + /// Parse a comma-separated collection where each item is parsed through + /// `parse_item` until the `end` token is met. + fn parse_collection( + &mut self, + end: Option, + parse_item: F + ) -> Spanned + where + C: FromIterator>, + F: FnMut(&mut Self) -> Result, (&'static str, Option)>, + { + self.parse_collection_comma_aware(end, parse_item).0 + } + /// Parse a comma-separated collection where each item is parsed through /// `parse_item` until the `end` token is met. The first item in the return /// tuple is the collection, the second item indicates whether the - /// collection can be coerced into a single item - /// (i.e. at least one comma appeared). - fn parse_collection_bracket_aware( + /// collection can be coerced into a single item (i.e. no comma appeared). + fn parse_collection_comma_aware( &mut self, end: Option, mut parse_item: F @@ -490,20 +499,6 @@ impl<'s> FuncParser<'s> { (Spanned::new(collection, Span { start, end }), can_be_coerced) } - /// Parse a comma-separated collection where each item is parsed through - /// `parse_item` until the `end` token is met. - fn parse_collection( - &mut self, - end: Option, - parse_item: F - ) -> Spanned - where - C: FromIterator>, - F: FnMut(&mut Self) -> Result, (&'static str, Option)>, - { - self.parse_collection_bracket_aware(end, parse_item).0 - } - /// Try to parse an identifier and do nothing if the peekable token is no /// identifier. fn parse_ident(&mut self) -> Option> { @@ -665,16 +660,16 @@ mod tests { fn Str(text: &str) -> Expr { Expr::Str(text.to_string()) } fn Pt(points: f32) -> Expr { Expr::Size(Size::pt(points)) } fn Neg(e1: Expr) -> Expr { Expr::Neg(Box::new(zspan(e1))) } - fn Add(e1: Expr, e2: Expr) -> Expr { + fn Add(e1: Expr, e2: Expr) -> Expr { Expr::Add(Box::new(zspan(e1)), Box::new(zspan(e2))) } - fn Sub(e1: Expr, e2: Expr) -> Expr { + fn Sub(e1: Expr, e2: Expr) -> Expr { Expr::Sub(Box::new(zspan(e1)), Box::new(zspan(e2))) } - fn Mul(e1: Expr, e2: Expr) -> Expr { + fn Mul(e1: Expr, e2: Expr) -> Expr { Expr::Mul(Box::new(zspan(e1)), Box::new(zspan(e2))) } - fn Div(e1: Expr, e2: Expr) -> Expr { + fn Div(e1: Expr, e2: Expr) -> Expr { Expr::Div(Box::new(zspan(e1)), Box::new(zspan(e2))) } diff --git a/src/syntax/tokens.rs b/src/syntax/tokens.rs index 6642a9bfa..3b34019d9 100644 --- a/src/syntax/tokens.rs +++ b/src/syntax/tokens.rs @@ -133,9 +133,9 @@ impl<'s> Token<'s> { ExprSize(_) => "size", ExprBool(_) => "bool", ExprHex(_) => "hex value", - Plus => "plus", - Hyphen => "minus", - Slash => "slash", + Plus => "plus", + Hyphen => "minus", + Slash => "slash", Star => "star", Underscore => "underscore", Backslash => "backslash", @@ -223,7 +223,7 @@ impl<'s> Iterator for Tokens<'s> { ':' if self.mode == Header => Colon, ',' if self.mode == Header => Comma, '=' if self.mode == Header => Equals, - + // Expression operators. '+' if self.mode == Header => Plus, '-' if self.mode == Header => Hyphen, From 38a24247422ed433a9743e7eacf5037ccf877fa0 Mon Sep 17 00:00:00 2001 From: Laurenz Date: Sat, 18 Jul 2020 18:02:38 +0200 Subject: [PATCH 4/4] =?UTF-8?q?Remove=20duplicate=20spans=20for=20func=20a?= =?UTF-8?q?rgs=20=E2=9D=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/func.rs | 2 +- src/syntax/func/mod.rs | 38 +++++++++++++++----------------------- src/syntax/parsing.rs | 20 +++++++------------- 3 files changed, 23 insertions(+), 37 deletions(-) diff --git a/src/func.rs b/src/func.rs index 2cdd026c2..33cc6e838 100644 --- a/src/func.rs +++ b/src/func.rs @@ -132,7 +132,7 @@ macro_rules! function { let func = $code; for arg in header.args.into_iter() { - feedback.errors.push(err!(arg.span(); "unexpected argument")); + feedback.errors.push(err!(arg.span; "unexpected argument")); } $crate::Pass::new(func, feedback) diff --git a/src/syntax/func/mod.rs b/src/syntax/func/mod.rs index 54d34531c..fd516208f 100644 --- a/src/syntax/func/mod.rs +++ b/src/syntax/func/mod.rs @@ -42,17 +42,22 @@ impl FuncArgs { } /// Add an argument. - pub fn add(&mut self, arg: FuncArg) { - match arg { - FuncArg::Pos(item) => self.pos.add(item), - FuncArg::Key(pair) => self.key.add(pair), + pub fn add(&mut self, arg: Spanned) { + match arg.v { + FuncArg::Pos(item) => self.pos.add(Spanned::new(item, arg.span)), + FuncArg::Key(pair) => self.key.add(Spanned::new(pair, arg.span)), } } /// Iterate over all arguments. - pub fn into_iter(self) -> impl Iterator { - self.pos.items.into_iter().map(|item| FuncArg::Pos(item)) - .chain(self.key.pairs.into_iter().map(|pair| FuncArg::Key(pair))) + pub fn into_iter(self) -> impl Iterator> { + let pos = self.pos.items.into_iter() + .map(|spanned| spanned.map(|item| FuncArg::Pos(item))); + + let key = self.key.pairs.into_iter() + .map(|spanned| spanned.map(|pair| FuncArg::Key(pair))); + + pos.chain(key) } } @@ -60,7 +65,7 @@ impl FromIterator> for FuncArgs { fn from_iter>>(iter: I) -> Self { let mut args = FuncArgs::new(); for item in iter.into_iter() { - args.add(item.v); + args.add(item); } args } @@ -70,22 +75,9 @@ impl FromIterator> for FuncArgs { #[derive(Debug, Clone, PartialEq)] pub enum FuncArg { /// A positional argument. - Pos(Spanned), + Pos(Expr), /// A keyword argument. - Key(Spanned), -} - -impl FuncArg { - /// The full span of this argument. - /// - /// In case of a positional argument this is just the span of the expression - /// and in case of a keyword argument the combined span of key and value. - pub fn span(&self) -> Span { - match self { - FuncArg::Pos(item) => item.span, - FuncArg::Key(item) => item.span, - } - } + Key(Pair), } /// Extra methods on [`Options`](Option) used for argument parsing. diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs index 07dfec7cb..9636df210 100644 --- a/src/syntax/parsing.rs +++ b/src/syntax/parsing.rs @@ -190,10 +190,8 @@ impl<'s> FuncParser<'s> { if let Some(ident) = p.parse_ident() { // This could still be a named tuple if let Some(Token::LeftParen) = p.peekv() { - let n_tuple = p.parse_named_tuple(ident) - .map(|t| Expr::NamedTuple(t)); - let span = n_tuple.span; - return Ok(Spanned::new(FuncArg::Pos(n_tuple), span)); + let tuple = p.parse_named_tuple(ident); + return Ok(tuple.map(|t| FuncArg::Pos(Expr::NamedTuple(t)))); } p.skip_whitespace(); @@ -210,22 +208,18 @@ impl<'s> FuncParser<'s> { // Add a keyword argument. let span = Span::merge(ident.span, value.span); - Ok(Spanned::new( - FuncArg::Key(Spanned::new(Pair { key: ident, value }, span)), - span, - )) + let pair = Pair { key: ident, value }; + Ok(Spanned::new(FuncArg::Key(pair), span)) } else { // Add a positional argument because there was no equals // sign after the identifier that could have been a key. - let ident = ident.map(|id| Expr::Ident(id)); - let span = ident.span; - Ok(Spanned::new(FuncArg::Pos(ident), span)) + Ok(ident.map(|id| FuncArg::Pos(Expr::Ident(id)))) } } else { // Add a positional argument because we haven't got an // identifier that could be an argument key. - p.parse_expr().map(|expr| Spanned { span: expr.span, v: FuncArg::Pos(expr) }) - .ok_or(("argument", None)) + let value = p.parse_expr().ok_or(("argument", None))?; + Ok(value.map(|expr| FuncArg::Pos(expr))) } }).v }