Do argument parsing ☑

src/macros.rs

@@ -62,3 +62,13 @@ macro_rules! pub_use_mod {
         pub use $name::*;
     };
 }
+
+/// Whether an expression matches a set of patterns.
+macro_rules! matches {
+    ($expression:expr, $( $pattern:pat )|+ $( if $guard: expr )?) => {
+        match $expression {
+            $( $pattern )|+ $( if $guard )? => true,
+            _ => false
+        }
+    }
+}

src/syntax/mod.rs

@@ -154,6 +154,22 @@ pub struct FuncArgs {
     pub keyword: Object,
 }
 
+#[derive(Debug, Clone, PartialEq)]
+pub enum Arg {
+    Pos(Spanned<Expression>),
+    Key(Pair),
+}
+
+impl Arg {
+    /// The span or the value or combined span of key and value.
+    pub fn span(&self) -> Span {
+        match self {
+            Arg::Pos(spanned) => spanned.span,
+            Arg::Key(Pair { key, value }) => Span::merge(key.span, value.span),
+        }
+    }
+}
+
 impl FuncArgs {
     pub fn new() -> FuncArgs {
         FuncArgs {
@@ -246,32 +262,40 @@ pub struct Colorization {
     pub tokens: Vec<Spanned<ColorToken>>,
 }
 
+impl Colorization {
+    pub fn new() -> Colorization {
+        Colorization { tokens: vec![] }
+    }
+
+    pub fn add(&mut self, token: ColorToken, span: Span) {
+        self.tokens.push(Spanned { v: token, span });
+    }
+
+    pub fn replace_last(&mut self, token: ColorToken) {
+        self.tokens.last_mut().expect("replace_last: no token").v = token;
+    }
+}
+
 /// Entities which can be colored by syntax highlighting.
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
 pub enum ColorToken {
     Comment,
-
     Bracket,
     FuncName,
     Colon,
-
     Key,
     Equals,
     Comma,
-
     Paren,
     Brace,
-
     ExprIdent,
     ExprStr,
     ExprNumber,
     ExprSize,
     ExprBool,
-
     Bold,
     Italic,
     Monospace,
-
     Invalid,
 }
 
@@ -279,3 +303,17 @@ pub enum ColorToken {
 pub struct ErrorMap {
     pub errors: Vec<Spanned<String>>,
 }
+
+impl ErrorMap {
+    pub fn new() -> ErrorMap {
+        ErrorMap { errors: vec![] }
+    }
+
+    pub fn add(&mut self, message: impl Into<String>, span: Span) {
+        self.errors.push(Spanned { v: message.into(), span });
+    }
+
+    pub fn add_at(&mut self, message: impl Into<String>, pos: Position) {
+        self.errors.push(Spanned { v: message.into(), span: Span::at(pos) })
+    }
+}

src/syntax/parsing.rs

@@ -20,7 +20,6 @@ struct Parser<'s> {
     ctx: ParseContext<'s>,
     colorization: Colorization,
     error_map: ErrorMap,
-
     tokens: Tokens<'s>,
     peeked: Option<Option<Spanned<Token<'s>>>>,
     position: Position,
@@ -32,9 +31,8 @@ impl<'s> Parser<'s> {
         Parser {
             src,
             ctx,
-            error_map: ErrorMap { errors: vec![] },
+            error_map: ErrorMap::new(),
-            colorization: Colorization { tokens: vec![] },
+            colorization: Colorization::new(),
-
             tokens: Tokens::new(src),
             peeked: None,
             position: Position::ZERO,
@@ -42,6 +40,7 @@ impl<'s> Parser<'s> {
         }
     }
 
+    /// The main parsing entrypoint.
     fn parse(mut self) -> (SyntaxTree, Colorization, ErrorMap) {
         let mut tree = SyntaxTree::new();
 
@@ -79,10 +78,18 @@ impl<'s> Parser<'s> {
         (tree, self.colorization, self.error_map)
     }
 
+    /// Parses a function including header and body with the cursor starting
+    /// right behind the first opening bracket.
     fn parse_func(&mut self) -> Option<Spanned<Node>> {
         let start = self.last_pos();
 
         let header = self.parse_func_header();
+        self.eat_until(|t| t == RightBracket, false);
+
+        if self.eat().map(Spanned::value) != Some(RightBracket) {
+            self.expected_at("closing bracket", self.pos());
+        }
+
         let call = self.parse_func_call(header)?;
 
         let end = self.pos();
@@ -91,21 +98,17 @@ impl<'s> Parser<'s> {
         Some(Spanned { v: Node::Func(call), span })
     }
 
+    /// Parses a function header including the closing bracket.
     fn parse_func_header(&mut self) -> Option<FuncHeader> {
         self.skip_whitespace();
-
+        let name = self.parse_func_name()?;
-        let name = self.parse_func_name().or_else(|| {
-            self.eat_until(|t| t == RightBracket, true);
-            None
-        })?;
 
         self.skip_whitespace();
         let args = match self.eat() {
             Some(Spanned { v: Colon, .. }) => self.parse_func_args(),
             Some(Spanned { v: RightBracket, .. }) => FuncArgs::new(),
             other => {
-                self.expected("colon or closing bracket", other);
+                self.expected_at("colon or closing bracket", name.span.end);
-                self.eat_until(|t| t == RightBracket, true);
                 FuncArgs::new()
             }
         };
@@ -113,8 +116,140 @@ impl<'s> Parser<'s> {
         Some(FuncHeader { name, args })
     }
 
+    /// Parses the function name if is the next token. Otherwise, it adds an
+    /// error and returns `None`.
+    fn parse_func_name(&mut self) -> Option<Spanned<Ident>> {
+        match self.eat() {
+            Some(Spanned { v: ExprIdent(ident), span }) => {
+                self.colorization.replace_last(ColorToken::FuncName);
+                return Some(Spanned { v: Ident(ident.to_string()), span });
+            }
+            other => self.expected_found_or_at("identifier", other, self.pos()),
+        }
+
+        None
+    }
+
+    /// Parses the function arguments and stops right before the final closing
+    /// bracket.
+    fn parse_func_args(&mut self) -> FuncArgs {
+        let mut args = FuncArgs::new();
+
+        loop {
+            self.skip_whitespace();
+            match self.peekv() {
+                Some(RightBracket) | None => break,
+                _ => match self.parse_arg() {
+                    Some(Arg::Pos(item)) => args.add_pos(item),
+                    Some(Arg::Key(pair)) => args.add_key_pair(pair),
+                    None => {}
+                }
+            }
+        }
+
+        args
+    }
+
+    /// Parse a positional or keyword argument.
+    fn parse_arg(&mut self) -> Option<Arg> {
+        let first = self.peek()?;
+        let span = first.span;
+
+        let arg = if let ExprIdent(ident) = first.v {
+            self.eat();
+            self.skip_whitespace();
+
+            let ident = Ident(ident.to_string());
+            if let Some(Equals) = self.peekv() {
+                self.colorization.replace_last(ColorToken::Key);
+
+                self.eat();
+                self.skip_whitespace();
+
+                self.parse_expr().map(|value| {
+                    Arg::Key(Pair {
+                        key: Spanned { v: ident, span },
+                        value,
+                    })
+                })
+            } else {
+                Some(Arg::Pos(Spanned::new(Expression::Ident(ident), span)))
+            }
+        } else {
+            self.parse_expr().map(|expr| Arg::Pos(expr))
+        };
+
+        if let Some(arg) = &arg {
+            self.skip_whitespace();
+            match self.peekv() {
+                Some(RightBracket) => {}
+                Some(Comma) => { self.eat(); }
+                Some(_) => self.expected_at("comma", arg.span().end),
+                _ => {}
+            }
+        } else {
+            let found = self.eat();
+            self.expected_found_or_at("value", found, self.pos());
+        }
+
+        arg
+    }
+
+    /// Parse a atomic or compound (tuple / object) expression.
+    fn parse_expr(&mut self) -> Option<Spanned<Expression>> {
+        let first = self.peek()?;
+        let mut expr = |v| {
+            self.eat();
+            Spanned { v, span: first.span }
+        };
+
+        Some(match first.v {
+            ExprIdent(i) => expr(Expression::Ident(Ident(i.to_string()))),
+            ExprStr(s) => expr(Expression::Str(s.to_string())),
+            ExprNumber(n) => expr(Expression::Number(n)),
+            ExprSize(s) => expr(Expression::Size(s)),
+            ExprBool(b) => expr(Expression::Bool(b)),
+            LeftParen => self.parse_tuple(),
+            LeftBrace => self.parse_object(),
+            _ => return None,
+        })
+    }
+
+    /// Parse a tuple expression.
+    fn parse_tuple(&mut self) -> Spanned<Expression> {
+        let start = self.pos();
+
+        // TODO: Do the thing.
+        self.eat_until(|t| matches!(t, RightParen | RightBracket), false);
+        if self.peekv() == Some(RightParen) {
+            self.eat();
+        }
+
+        let end = self.pos();
+        let span = Span { start, end };
+
+        Spanned { v: Expression::Tuple(Tuple::new()), span }
+    }
+
+    /// Parse an object expression.
+    fn parse_object(&mut self) -> Spanned<Expression> {
+        let start = self.pos();
+
+        // TODO: Do the thing.
+        self.eat_until(|t| matches!(t, RightBrace | RightBracket), false);
+        if self.peekv() == Some(RightBrace) {
+            self.eat();
+        }
+
+        let end = self.pos();
+        let span = Span { start, end };
+
+        Spanned { v: Expression::Object(Object::new()), span }
+    }
+
+    /// Parse the body of a function invocation.
     fn parse_func_call(&mut self, header: Option<FuncHeader>) -> Option<FuncCall> {
-        let body = if self.peek() == Some(LeftBracket) {
+        let body = if self.peekv() == Some(LeftBracket) {
             self.eat();
 
             let start = self.tokens.index();
@@ -127,9 +262,10 @@ impl<'s> Parser<'s> {
             let body = &self.src[start .. end];
 
             if found {
-                assert_eq!(self.eat().map(Spanned::value), Some(RightBracket));
+                let next = self.eat().map(Spanned::value);
+                debug_assert_eq!(next, Some(RightBracket));
             } else {
-                self.error_here("expected closing bracket");
+                self.expected_at("closing bracket", self.pos());
             }
 
             Some(body)
@@ -139,111 +275,57 @@ impl<'s> Parser<'s> {
 
         let header = header?;
         let parser = self.ctx.scope.get_parser(header.name.v.as_str()).or_else(|| {
-            self.error(
+            let message = format!("unknown function: `{}`", header.name.v);
-                format!("unknown function: `{}`", header.name.v),
+            self.error_map.add(message, header.name.span);
-                header.name.span
-            );
             None
         })?;
 
         Some(FuncCall(parser(header, body, self.ctx).unwrap()))
     }
 
-    fn parse_func_name(&mut self) -> Option<Spanned<Ident>> {
+    /// Skip all whitespace/comment tokens.
-        match self.eat() {
-            Some(Spanned { v: ExprIdent(ident), span }) => {
-                self.color(Spanned { v: ColorToken::FuncName, span }, true);
-                Some(Spanned { v: Ident(ident.to_string()), span })
-            }
-            other => {
-                self.expected("identifier", other);
-                None
-            }
-        }
-    }
-
-    fn parse_func_args(&mut self) -> FuncArgs {
-        // todo!()
-        self.eat_until(|t| t == RightBracket, true);
-        FuncArgs::new()
-    }
-
-    fn parse_tuple(&mut self) -> Spanned<Expression> {
-        todo!("parse_tuple")
-    }
-
-    fn parse_object(&mut self) -> Spanned<Expression> {
-        todo!("parse_object")
-    }
-
     fn skip_whitespace(&mut self) {
-        self.eat_until(|t| match t {
+        self.eat_until(|t|
-            Whitespace(_) | LineComment(_) | BlockComment(_) => false,
+            !matches!(t, Whitespace(_) | LineComment(_) | BlockComment(_)), false)
-            _ => true,
-        }, false)
     }
 
-    fn expected(&mut self, thing: &str, found: Option<Spanned<Token>>) {
+    /// Add an error about an `thing` which was expected but not found at the
-        if let Some(Spanned { v: found, span }) = found {
+    /// given position.
-            self.error(
+    fn expected_at(&mut self, thing: &str, pos: Position) {
-                format!("expected {}, found {}", thing, name(found)),
+        self.error_map.add_at(format!("expected {}", thing), pos);
-                span
+    }
-            );
+
-        } else {
+    /// Add an error about an expected `thing` which was not found, showing
-            self.error_here(format!("expected {}", thing));
+    /// what was found instead.
+    fn expected_found(&mut self, thing: &str, found: Spanned<Token>) {
+        let message = format!("expected {}, found {}", thing, name(found.v));
+        self.error_map.add(message, found.span);
+    }
+
+    /// Add a found-error if `found` is some and a positional error, otherwise.
+    fn expected_found_or_at(
+        &mut self,
+        thing: &str,
+        found: Option<Spanned<Token>>,
+        pos: Position
+    ) {
+        match found {
+            Some(found) => self.expected_found(thing, found),
+            None => self.expected_at(thing, pos),
         }
     }
 
+    /// Add an error about an unexpected token `found`.
     fn unexpected(&mut self, found: Spanned<Token>) {
-        self.error_map.errors.push(found.map(|t| format!("unexpected {}", name(t))));
+        self.error_map.add(format!("unexpected {}", name(found.v)), found.span);
-    }
-
-    fn error(&mut self, message: impl Into<String>, span: Span) {
-        self.error_map.errors.push(Spanned { v: message.into(), span });
-    }
-
-    fn error_here(&mut self, message: impl Into<String>) {
-        self.error(message, Span::at(self.pos()));
-    }
-
-    fn color(&mut self, token: Spanned<ColorToken>, replace_last: bool) {
-        if replace_last {
-            if let Some(last) = self.colorization.tokens.last_mut() {
-                *last = token;
-                return;
-            }
-        }
-
-        self.colorization.tokens.push(token);
-    }
-
-    fn color_token(&mut self, token: Spanned<Token<'s>>) {
-        let colored = match token.v {
-            LineComment(_) | BlockComment(_) => Some(ColorToken::Comment),
-            StarSlash                  => Some(ColorToken::Invalid),
-            LeftBracket | RightBracket => Some(ColorToken::Bracket),
-            LeftParen | RightParen     => Some(ColorToken::Paren),
-            LeftBrace | RightBrace     => Some(ColorToken::Brace),
-            Colon         => Some(ColorToken::Colon),
-            Comma         => Some(ColorToken::Comma),
-            Equals        => Some(ColorToken::Equals),
-            ExprIdent(_)  => Some(ColorToken::ExprIdent),
-            ExprStr(_)    => Some(ColorToken::ExprStr),
-            ExprNumber(_) => Some(ColorToken::ExprNumber),
-            ExprSize(_)   => Some(ColorToken::ExprSize),
-            ExprBool(_)   => Some(ColorToken::ExprBool),
-            _ => None,
-        };
-
-        if let Some(color) = colored {
-            self.colorization.tokens.push(Spanned { v: color, span: token.span });
-        }
     }
 
+    /// Consume tokens until the function returns true and only consume the last
+    /// token if instructed to.
     fn eat_until<F>(&mut self, mut f: F, eat_match: bool)
     where F: FnMut(Token<'s>) -> bool {
         while let Some(token) = self.peek() {
-            if f(token) {
+            if f(token.v) {
                 if eat_match {
                     self.eat();
                 }
@@ -254,56 +336,90 @@ impl<'s> Parser<'s> {
         }
     }
 
+    /// Consume and return the next token, update positions and colorize the
+    /// token. All colorable tokens are per default colorized here, to override
+    /// a colorization use `Colorization::replace_last`.
     fn eat(&mut self) -> Option<Spanned<Token<'s>>> {
-        let token = self.peeked.take().unwrap_or_else(|| self.tokens.next());
+        let token = self.peeked.take()
+            .unwrap_or_else(|| self.tokens.next());
 
-        self.last_position = self.position;
+        if let Some(token) = token {
-        if let Some(spanned) = token {
+            if let Some(color) = color(token.v) {
-            self.color_token(spanned);
+                self.colorization.add(color, token.span);
-            self.position = spanned.span.end;
+            }
+
+            self.last_position = self.position;
+            self.position = token.span.end;
         }
 
         token
     }
 
-    fn peek(&mut self) -> Option<Token<'s>> {
+    /// Peek at the next token without consuming it.
+    fn peek(&mut self) -> Option<Spanned<Token<'s>>> {
         let iter = &mut self.tokens;
-        self.peeked
+        *self.peeked.get_or_insert_with(|| iter.next())
-            .get_or_insert_with(|| iter.next())
-            .map(Spanned::value)
     }
 
+    fn peekv(&mut self) -> Option<Token<'s>> {
+        self.peek().map(Spanned::value)
+    }
+
+    /// The position at the end of the last eat token / start of the peekable
+    /// token.
     fn pos(&self) -> Position {
         self.position
     }
 
+    /// The position at the start of the last eaten token.
     fn last_pos(&self) -> Position {
         self.last_position
     }
 }
 
+/// The name of a token in an `expected <...>` error.
 fn name(token: Token) -> &'static str {
     match token {
         Whitespace(_) => "whitespace",
         LineComment(_) | BlockComment(_) => "comment",
-        StarSlash => "end of block comment",
+        StarSlash     => "end of block comment",
-        LeftBracket => "opening bracket",
+        LeftBracket   => "opening bracket",
-        RightBracket => "closing bracket",
+        RightBracket  => "closing bracket",
-        LeftParen => "opening paren",
+        LeftParen     => "opening paren",
-        RightParen => "closing paren",
+        RightParen    => "closing paren",
-        LeftBrace => "opening brace",
+        LeftBrace     => "opening brace",
-        RightBrace => "closing brace",
+        RightBrace    => "closing brace",
-        Colon => "colon",
+        Colon         => "colon",
-        Comma => "comma",
+        Comma         => "comma",
-        Equals => "equals sign",
+        Equals        => "equals sign",
-        ExprIdent(_) => "identifier",
+        ExprIdent(_)  => "identifier",
-        ExprStr(_) => "string",
+        ExprStr(_)    => "string",
         ExprNumber(_) => "number",
-        ExprSize(_) => "size",
+        ExprSize(_)   => "size",
-        ExprBool(_) => "bool",
+        ExprBool(_)   => "bool",
-        Star => "star",
+        Star          => "star",
-        Underscore => "underscore",
+        Underscore    => "underscore",
-        Backtick => "backtick",
+        Backtick      => "backtick",
-        Text(_) => "text",
+        Text(_)       => "invalid identifier",
     }
 }
+
+/// The color token corresponding to a token.
+fn color(token: Token) -> Option<ColorToken> {
+    Some(match token {
+        LineComment(_) | BlockComment(_) => ColorToken::Comment,
+        LeftBracket    | RightBracket    => ColorToken::Bracket,
+        LeftParen      | RightParen      => ColorToken::Paren,
+        LeftBrace      | RightBrace      => ColorToken::Brace,
+        Colon         => ColorToken::Colon,
+        Comma         => ColorToken::Comma,
+        Equals        => ColorToken::Equals,
+        ExprIdent(_)  => ColorToken::ExprIdent,
+        ExprStr(_)    => ColorToken::ExprStr,
+        ExprNumber(_) => ColorToken::ExprNumber,
+        ExprSize(_)   => ColorToken::ExprSize,
+        ExprBool(_)   => ColorToken::ExprBool,
+        StarSlash     => ColorToken::Invalid,
+        _ => return None,
+    })
+}

tests/parser/trees.rs

@@ -1,15 +1,19 @@
+// Basics.
 p "" => []
 p "hi" => [T("hi")]
 p "hi you" => [T("hi"), S, T("you")]
 p "❤\n\n 🌍" => [T("❤"), N, T("🌍")]
 
+// Functions.
 p "[func]" => [func!("func"; None)]
 p "[tree][hi *you*]" => [func!("tree"; Some([T("hi"), S, B, T("you"), B]))]
-
 p "from [align: left] to" => [
     T("from"), S, func!("align", pos: [ID("left")]; None), S, T("to"),
 ]
-
+p "[f: left, 12pt, false]" => [
+    func!("f", pos: [ID("left"), SIZE(Size::pt(12.0)), BOOL(false)]; None)
+]
+p "[f: , hi, * \"du\"]" => [func!("f", pos: [ID("hi"), STR("du")]; None)]
 p "[box: x=1.2pt, false][a b c] bye" => [
     func!(
         "box",
@@ -20,14 +24,23 @@ p "[box: x=1.2pt, false][a b c] bye" => [
     S, T("bye"),
 ]
 
-c "hi" => []
+// Errors.
-c "[align: left][\n    _body_\n]" => [
+e "[f: , hi, * \"du\"]" => [
-    (0:0, 0:1, B),
+    (0:4,  0:5,  "expected value, found comma"),
-    (0:1, 0:6, FN),
+    (0:10, 0:11, "expected value, found invalid identifier"),
-    (0:6, 0:7, CL),
+]
-    (0:8, 0:12, ID),
+e "[f:, , ,]" => [
-    (0:12, 0:13, B),
+    (0:3, 0:4, "expected value, found comma"),
-    (0:13, 0:14, B),
+    (0:5, 0:6, "expected value, found comma"),
-    (1:4, 1:10, IT),
+    (0:7, 0:8, "expected value, found comma"),
-    (2:0, 2:2, B),
+]
+e "[f:" => [(0:3, 0:3, "expected closing bracket")]
+e "[f: hi" => [(0:6, 0:6, "expected closing bracket")]
+e "[f: hey   12pt]" => [(0:7, 0:7, "expected comma")]
+e "[box: x=, false y=z=4" => [
+    (0:8,  0:9,  "expected value, found comma"),
+    (0:15, 0:15, "expected comma"),
+    (0:19, 0:19, "expected comma"),
+    (0:19, 0:20, "expected value, found equals sign"),
+    (0:21, 0:21, "expected closing bracket"),
 ]

tests/src/parser.rs

@@ -40,7 +40,7 @@ fn test(tests: Vec<(&str, Vec<Case>)>) {
                 Case::Okay => okay += 1,
                 Case::Failed { line, src, expected, found } => {
                     println!();
-                    println!(" - Case failed in file {}.rs in line {}.", file, line);
+                    println!(" ❌  Case failed in file {}.rs in line {}.", file, line);
                     println!("   - Source:   {:?}", src);
                     println!("   - Expected: {}", expected);
                     println!("   - Found:    {}", found);
@@ -119,7 +119,11 @@ macro_rules! case {
     });
 
     (e $src:expr, [$($e:tt)*]) => ({
-        let expected = ErrorMap { errors: list!([$($e)*]) };
+        let errors = list!([$($e)*]).into_iter()
+            .map(|s| s.map(|m| m.to_string()))
+            .collect();
+
+        let expected = ErrorMap { errors };
         let found = parse($src, ParseContext { scope: &scope() }).2;
         (expected == found, expected, found)
     });
@@ -204,8 +208,10 @@ function! {
     }
 
     parse(header, body, ctx) {
+        let cloned = header.clone();
+        header.args.clear();
         DebugFn {
-            header: header.clone(),
+            header: cloned,
             body: parse!(optional: body, ctx),
         }
     }