Decoupled function parser 🔗 [WIP]

.gitignore

@@ -3,3 +3,4 @@
 **/*.rs.bk
 Cargo.lock
 tests/cache
+_things

src/bin/main.rs

@@ -41,7 +41,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
     let provider = FileSystemFontProvider::from_index("../fonts/index.json").unwrap();
     typesetter.add_font_provider(provider);
 
-    let layouts = block_on(typesetter.typeset(&src))?;
+    let layouts = block_on(typesetter.typeset(&src));
 
     let exporter = PdfExporter::new();
     let writer = BufWriter::new(File::create(&dest)?);

src/error.rs

@@ -1,10 +1,20 @@
-#[derive(Debug, Clone, Eq, PartialEq)]
+use serde::Serialize;
+
+
+#[derive(Debug, Clone, Eq, PartialEq, Serialize)]
 pub struct Error {
     pub message: String,
+    pub severity: Severity,
 }
 
 impl Error {
-    pub fn new(message: impl Into<String>) -> Error {
+    pub fn new(message: impl Into<String>, severity: Severity) -> Error {
-        Error { message: message.into() }
+        Error { message: message.into(), severity }
     }
 }
+
+#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize)]
+pub enum Severity {
+    Warning,
+    Error,
+}

src/func/macros.rs

@@ -22,7 +22,7 @@ macro_rules! function {
 
     // Parse trait.
     (@parse($($a:tt)*) parse(default) $($r:tt)*) => {
-        function!(@parse($($a)*) parse() { Default::default() } $($r)*);
+        function!(@parse($($a)*) parse(_h, _b, _c, _e, _d, _m) {Default::default() } $($r)*);
     };
     (@parse($($a:tt)*) parse($h:ident, $b:ident, $c:ident, $e:ident, $d:ident) $($r:tt)* ) => {
         function!(@parse($($a)*) parse($h, $b, $c, $e, $d, _metadata) $($r)*);
@@ -40,7 +40,7 @@ macro_rules! function {
 
             fn parse(
                 #[allow(unused)] mut $header: FuncHeader,
-                #[allow(unused)] $body: Option<Spanned<&str>>,
+                #[allow(unused)] $body: Option<(Position, &str)>,
                 #[allow(unused)] $ctx: ParseContext,
                 #[allow(unused)] $metadata: Self::Meta,
             ) -> Parsed<Self> where Self: Sized {
@@ -85,7 +85,7 @@ macro_rules! function {
 macro_rules! body {
     (opt: $body:expr, $ctx:expr, $errors:expr, $decos:expr) => ({
         $body.map(|body| {
-            let parsed = $crate::syntax::parse(body.span.start, body.v, $ctx);
+            let parsed = $crate::syntax::parse(body.0, body.1, $ctx);
             $errors.extend(parsed.errors);
             $decos.extend(parsed.decorations);
             parsed.output
@@ -99,13 +99,19 @@ macro_rules! body {
     };
 }
 
-/// Construct a spanned error.
+/// Construct an error with an optional span.
 #[macro_export]
 macro_rules! err {
-    ($span:expr, $($args:tt)*) => {
+    (@$severity:ident: $span:expr; $($args:tt)*) => {
-        $crate::syntax::Spanned {
+        $crate::syntax::Spanned { v: err!(@Error: $($args)*), span: $span }
-            v: $crate::error::Error::new(format!($($args)*)),
+    };
-            span: $span,
+
+    (@$severity:ident: $($args:tt)*) => {
+        $crate::error::Error {
+            message: format!($($args)*),
+            severity: $crate::error::Severity::$severity,
         }
     };
+
+    ($($tts:tt)*) => { err!(@Error: $($tts)*) };
 }

src/func/mod.rs

@@ -1,6 +1,5 @@
 //! Dynamic typesetting functions.
 
-use std::any::Any;
 use std::collections::HashMap;
 use std::fmt::{self, Debug, Formatter};
 
@@ -12,6 +11,7 @@ mod macros;
 /// Useful imports for creating your own functions.
 pub mod prelude {
     pub use super::{Scope, Parse, Command, Commands};
+    pub use crate::error::Error;
     pub use crate::layout::prelude::*;
     pub use crate::syntax::prelude::*;
     pub use crate::size::{Size, Size2D, SizeBox, ValueBox, ScaleSize, FSize, PSize};
@@ -26,7 +26,7 @@ pub trait Parse {
     /// Parse the header and body into this function given a context.
     fn parse(
         header: FuncHeader,
-        body: Option<Spanned<&str>>,
+        body: Option<(Position, &str)>,
         ctx: ParseContext,
         metadata: Self::Meta,
     ) -> Parsed<Self> where Self: Sized;
@@ -36,7 +36,7 @@ pub trait Parse {
 /// implements [`Model`].
 type Parser = dyn Fn(
     FuncHeader,
-    Option<Spanned<&str>>,
+    Option<(Position, &str)>,
     ParseContext,
 ) -> Parsed<Box<dyn Model>>;
 
@@ -102,11 +102,16 @@ impl Scope {
     }
 
     /// Return the parser with the given name if there is one.
-    pub(crate) fn get_parser(&self, name: &str) -> Result<&Parser, &Parser> {
+    pub fn get_parser(&self, name: &str) -> Result<&Parser, &Parser> {
         self.parsers.get(name)
             .map(|x| &**x)
             .ok_or_else(|| &*self.fallback)
     }
+
+    /// Return the fallback parser.
+    pub fn get_fallback_parser(&self) -> &Parser {
+        &*self.fallback
+    }
 }
 
 impl Debug for Scope {

src/layout/mod.rs

@@ -5,7 +5,7 @@ use smallvec::SmallVec;
 use toddle::query::{SharedFontLoader, FontIndex};
 
 use crate::error::Error;
-use crate::syntax::SpanVec;
+use crate::syntax::{SyntaxModel, SpanVec};
 use crate::size::{Size, Size2D, SizeBox};
 use crate::style::LayoutStyle;
 
@@ -26,7 +26,7 @@ pub mod prelude {
 
 /// Different kinds of layouters (fully re-exported).
 pub mod layouters {
-    pub use super::model::layout;
+    pub use super::model::ModelLayouter;
     pub use super::line::{LineLayouter, LineContext};
     pub use super::stack::{StackLayouter, StackContext};
     pub use super::text::{layout_text, TextContext};
@@ -37,20 +37,6 @@ pub use self::layouters::*;
 pub use self::prelude::*;
 
 
-pub struct Layouted<T> {
-    pub output: T,
-    pub errors: SpanVec<Error>,
-}
-
-impl<T> Layouted<T> {
-    pub fn map<F, U>(self, f: F) -> Layouted<U> where F: FnOnce(T) -> U {
-        Layouted {
-            output: f(self.output),
-            errors: self.errors,
-        }
-    }
-}
-
 /// A collection of layouts.
 pub type MultiLayout = Vec<Layout>;
 
@@ -80,34 +66,6 @@ impl Layout {
     }
 }
 
-/// Layout components that can be serialized.
-pub trait Serialize {
-    /// Serialize the data structure into an output writable.
-    fn serialize<W: Write>(&self, f: &mut W) -> io::Result<()>;
-}
-
-impl Serialize for Layout {
-    fn serialize<W: Write>(&self, f: &mut W) -> io::Result<()> {
-        writeln!(f, "{:.4} {:.4}", self.dimensions.x.to_pt(), self.dimensions.y.to_pt())?;
-        writeln!(f, "{}", self.actions.len())?;
-        for action in &self.actions {
-            action.serialize(f)?;
-            writeln!(f)?;
-        }
-        Ok(())
-    }
-}
-
-impl Serialize for MultiLayout {
-    fn serialize<W: Write>(&self, f: &mut W) -> io::Result<()> {
-        writeln!(f, "{}", self.len())?;
-        for layout in self {
-            layout.serialize(f)?;
-        }
-        Ok(())
-    }
-}
-
 /// The general context for layouting.
 #[derive(Debug, Clone)]
 pub struct LayoutContext<'a, 'p> {
@@ -133,6 +91,26 @@ pub struct LayoutContext<'a, 'p> {
     pub debug: bool,
 }
 
+pub struct Layouted<T> {
+    pub output: T,
+    pub errors: SpanVec<Error>,
+}
+
+impl<T> Layouted<T> {
+    pub fn map<F, U>(self, f: F) -> Layouted<U> where F: FnOnce(T) -> U {
+        Layouted {
+            output: f(self.output),
+            errors: self.errors,
+        }
+    }
+}
+
+pub async fn layout(model: &SyntaxModel, ctx: LayoutContext<'_, '_>) -> Layouted<MultiLayout> {
+    let mut layouter = ModelLayouter::new(ctx);
+    layouter.layout_syntax_model(model).await;
+    layouter.finish()
+}
+
 /// A possibly stack-allocated vector of layout spaces.
 pub type LayoutSpaces = SmallVec<[LayoutSpace; 2]>;
 
@@ -397,3 +375,31 @@ impl LastSpacing {
         }
     }
 }
+
+/// Layout components that can be serialized.
+pub trait Serialize {
+    /// Serialize the data structure into an output writable.
+    fn serialize<W: Write>(&self, f: &mut W) -> io::Result<()>;
+}
+
+impl Serialize for Layout {
+    fn serialize<W: Write>(&self, f: &mut W) -> io::Result<()> {
+        writeln!(f, "{:.4} {:.4}", self.dimensions.x.to_pt(), self.dimensions.y.to_pt())?;
+        writeln!(f, "{}", self.actions.len())?;
+        for action in &self.actions {
+            action.serialize(f)?;
+            writeln!(f)?;
+        }
+        Ok(())
+    }
+}
+
+impl Serialize for MultiLayout {
+    fn serialize<W: Write>(&self, f: &mut W) -> io::Result<()> {
+        writeln!(f, "{}", self.len())?;
+        for layout in self {
+            layout.serialize(f)?;
+        }
+        Ok(())
+    }
+}

src/layout/model.rs

@@ -1,5 +1,3 @@
-use std::pin::Pin;
-use std::future::Future;
 use smallvec::smallvec;
 
 use crate::error::Error;
@@ -9,17 +7,8 @@ use crate::syntax::{SpanVec, Spanned, Span, offset_spans};
 use super::*;
 
 
-pub async fn layout(
-    model: &SyntaxModel,
-    ctx: LayoutContext<'_, '_>
-) -> Layouted<MultiLayout> {
-    let mut layouter = ModelLayouter::new(ctx);
-    layouter.layout_syntax_model(model).await;
-    layouter.finish()
-}
-
 #[derive(Debug, Clone)]
-struct ModelLayouter<'a, 'p> {
+pub struct ModelLayouter<'a, 'p> {
     ctx: LayoutContext<'a, 'p>,
     layouter: LineLayouter,
     style: LayoutStyle,
@@ -28,7 +17,7 @@ struct ModelLayouter<'a, 'p> {
 
 impl<'a, 'p> ModelLayouter<'a, 'p> {
     /// Create a new syntax tree layouter.
-    fn new(ctx: LayoutContext<'a, 'p>) -> ModelLayouter<'a, 'p> {
+    pub fn new(ctx: LayoutContext<'a, 'p>) -> ModelLayouter<'a, 'p> {
         ModelLayouter {
             layouter: LineLayouter::new(LineContext {
                 spaces: ctx.spaces.clone(),
@@ -44,7 +33,7 @@ impl<'a, 'p> ModelLayouter<'a, 'p> {
         }
     }
 
-    fn layout<'r>(
+    pub fn layout<'r>(
         &'r mut self,
         model: Spanned<&'r dyn Model>
     ) -> DynFuture<'r, ()> { Box::pin(async move {
@@ -60,75 +49,11 @@ impl<'a, 'p> ModelLayouter<'a, 'p> {
         self.errors.extend(offset_spans(layouted.errors, model.span.start));
 
         for command in commands {
-            self.execute_command(command, model.span);
+            self.execute_command(command, model.span).await;
         }
     }) }
 
-    fn execute_command<'r>(
+    pub fn layout_syntax_model<'r>(
-        &'r mut self,
-        command: Command<'r>,
-        model_span: Span,
-    ) -> DynFuture<'r, ()> { Box::pin(async move {
-        use Command::*;
-
-        match command {
-            LayoutSyntaxModel(model) => self.layout_syntax_model(model).await,
-
-            Add(layout) => self.layouter.add(layout),
-            AddMultiple(layouts) => self.layouter.add_multiple(layouts),
-            SpacingFunc(space, kind, axis) => match axis {
-                Primary => self.layouter.add_primary_spacing(space, kind),
-                Secondary => self.layouter.add_secondary_spacing(space, kind),
-            }
-
-            FinishLine => self.layouter.finish_line(),
-            FinishSpace => self.layouter.finish_space(true),
-            BreakParagraph => self.layout_paragraph(),
-            BreakPage => {
-                if self.ctx.nested {
-                    self.errors.push(Spanned::new(
-                        Error::new( "page break cannot be issued from nested context"),
-                        model_span,
-                    ));
-                } else {
-                    self.layouter.finish_space(true)
-                }
-            }
-
-            SetTextStyle(style) => {
-                self.layouter.set_line_spacing(style.line_spacing());
-                self.style.text = style;
-            }
-            SetPageStyle(style) => {
-                if self.ctx.nested {
-                    self.errors.push(Spanned::new(
-                        Error::new("page style cannot be changed from nested context"),
-                        model_span,
-                    ));
-                } else {
-                    self.style.page = style;
-
-                    let margins = style.margins();
-                    self.ctx.base = style.dimensions.unpadded(margins);
-                    self.layouter.set_spaces(smallvec![
-                        LayoutSpace {
-                            dimensions: style.dimensions,
-                            padding: margins,
-                            expansion: LayoutExpansion::new(true, true),
-                        }
-                    ], true);
-                }
-            }
-
-            SetAlignment(alignment) => self.ctx.alignment = alignment,
-            SetAxes(axes) => {
-                self.layouter.set_axes(axes);
-                self.ctx.axes = axes;
-            }
-        }
-    }) }
-
-    fn layout_syntax_model<'r>(
         &'r mut self,
         model: &'r SyntaxModel
     ) -> DynFuture<'r, ()> { Box::pin(async move {
@@ -161,6 +86,73 @@ impl<'a, 'p> ModelLayouter<'a, 'p> {
         }
     }) }
 
+    pub fn finish(self) -> Layouted<MultiLayout> {
+        Layouted {
+            output: self.layouter.finish(),
+            errors: self.errors,
+        }
+    }
+
+    fn execute_command<'r>(
+        &'r mut self,
+        command: Command<'r>,
+        span: Span,
+    ) -> DynFuture<'r, ()> { Box::pin(async move {
+        use Command::*;
+
+        match command {
+            LayoutSyntaxModel(model) => self.layout_syntax_model(model).await,
+
+            Add(layout) => self.layouter.add(layout),
+            AddMultiple(layouts) => self.layouter.add_multiple(layouts),
+            SpacingFunc(space, kind, axis) => match axis {
+                Primary => self.layouter.add_primary_spacing(space, kind),
+                Secondary => self.layouter.add_secondary_spacing(space, kind),
+            }
+
+            FinishLine => self.layouter.finish_line(),
+            FinishSpace => self.layouter.finish_space(true),
+            BreakParagraph => self.layout_paragraph(),
+            BreakPage => {
+                if self.ctx.nested {
+                    self.errors.push(err!(span;
+                        "page break cannot be issued from nested context"));
+                } else {
+                    self.layouter.finish_space(true)
+                }
+            }
+
+            SetTextStyle(style) => {
+                self.layouter.set_line_spacing(style.line_spacing());
+                self.style.text = style;
+            }
+            SetPageStyle(style) => {
+                if self.ctx.nested {
+                    self.errors.push(err!(span;
+                        "page style cannot be changed from nested context"));
+                } else {
+                    self.style.page = style;
+
+                    let margins = style.margins();
+                    self.ctx.base = style.dimensions.unpadded(margins);
+                    self.layouter.set_spaces(smallvec![
+                        LayoutSpace {
+                            dimensions: style.dimensions,
+                            padding: margins,
+                            expansion: LayoutExpansion::new(true, true),
+                        }
+                    ], true);
+                }
+            }
+
+            SetAlignment(alignment) => self.ctx.alignment = alignment,
+            SetAxes(axes) => {
+                self.layouter.set_axes(axes);
+                self.ctx.axes = axes;
+            }
+        }
+    }) }
+
     async fn layout_text(&mut self, text: &str) {
         self.layouter.add(layout_text(text, TextContext {
             loader: &self.ctx.loader,
@@ -183,11 +175,4 @@ impl<'a, 'p> ModelLayouter<'a, 'p> {
             SpacingKind::PARAGRAPH,
         );
     }
-
-    fn finish(self) -> Layouted<MultiLayout> {
-        Layouted {
-            output: self.layouter.finish(),
-            errors: self.errors,
-        }
-    }
 }

src/lib.rs

@@ -24,10 +24,9 @@ use std::cell::RefCell;
 use smallvec::smallvec;
 
 use toddle::query::{FontLoader, FontProvider, SharedFontLoader};
-use toddle::Error as FontError;
 
 use crate::func::Scope;
-use crate::layout::{Layouted, LayoutContext, MultiLayout};
+use crate::layout::{Layouted, MultiLayout};
 use crate::syntax::{parse, ParseContext, Parsed, SyntaxModel, Position};
 use crate::style::{LayoutStyle, PageStyle, TextStyle};
 

src/library/maps/mod.rs

@@ -21,8 +21,8 @@ macro_rules! key {
         impl ExpressionKind for $type {
             const NAME: &'static str = $name;
 
-            fn from_expr(expr: Spanned<Expression>) -> ParseResult<Self> {
+            fn from_expr(expr: Spanned<Expr>) -> ParseResult<Self> {
-                if let Expression::Ident(ident) = expr.v {
+                if let Expr::Ident(ident) = expr.v {
                     Self::from_ident(&Spanned::new(ident, expr.span))
                 } else {
                     error!("expected {}", Self::NAME);
@@ -55,8 +55,8 @@ impl<T> Into<Option<T>> for DefaultKey<T> {
 impl<T> ExpressionKind for DefaultKey<T> where T: ExpressionKind {
     const NAME: &'static str = T::NAME;
 
-    fn from_expr(expr: Spanned<Expression>) -> ParseResult<DefaultKey<T>> {
+    fn from_expr(expr: Spanned<Expr>) -> ParseResult<DefaultKey<T>> {
-        if let Expression::Ident(ident) = &expr.v {
+        if let Expr::Ident(ident) = &expr.v {
             match ident.as_str() {
                 "default" => return Ok(DefaultKey::None),
                 _ => {},

src/library/mod.rs

@@ -63,8 +63,8 @@ function! {
             body: parse!(optional: body, ctx),
             list: {
                 header.args.iter_pos().map(|arg| match arg.v {
-                    Expression::Str(s) |
+                    Expr::Str(s) |
-                    Expression::Ident(Ident(s)) => Ok(s.to_lowercase()),
+                    Expr::Ident(Ident(s)) => Ok(s.to_lowercase()),
                     _ => error!("expected identifier or string"),
                 }).collect::<LayoutResult<Vec<_>>>()?
             }
@@ -117,8 +117,8 @@ function! {
     parse(header, body, ctx) {
         FontWeightFunc {
             body: parse!(optional: body, ctx),
-            weight: match header.args.get_pos::<Expression>()? {
+            weight: match header.args.get_pos::<Expr>()? {
-                Expression::Number(weight) => {
+                Expr::Number(weight) => {
                     let weight = weight.round() as i16;
                     FontWeight(
                         if weight < 100 { 100 }
@@ -126,7 +126,7 @@ function! {
                         else { 900 }
                     )
                 }
-                Expression::Ident(Ident(s)) => {
+                Expr::Ident(Ident(s)) => {
                     match FontWeight::from_str(&s) {
                         Some(weight) => weight,
                         None => error!("invalid font weight: `{}`", s),
@@ -263,7 +263,7 @@ function! {
             SpacingFunc {
                 axis: AxisKey::Specific(axis),
                 spacing: FSize::from_expr(
-                    header.args.get_pos::<Spanned<Expression>>()?
+                    header.args.get_pos::<Spanned<Expr>>()?
                 )?,
             }
         } else {

src/syntax/expr.rs

@@ -4,7 +4,7 @@ use super::*;
 
 /// An argument or return value.
 #[derive(Clone, PartialEq)]
-pub enum Expression {
+pub enum Expr {
     Ident(Ident),
     Str(String),
     Number(f64),
@@ -14,9 +14,24 @@ pub enum Expression {
     Object(Object),
 }
 
-impl Display for Expression {
+impl Expr {
+    pub fn name(&self) -> &'static str {
+        use Expr::*;
+        match self {
+            Ident(_) => "identifier",
+            Str(_) => "string",
+            Number(_) => "number",
+            Size(_) => "size",
+            Bool(_) => "boolean",
+            Tuple(_) => "tuple",
+            Object(_) => "object",
+        }
+    }
+}
+
+impl Display for Expr {
     fn fmt(&self, f: &mut Formatter) -> fmt::Result {
-        use Expression::*;
+        use Expr::*;
         match self {
             Ident(i) => write!(f, "{}", i),
             Str(s) => write!(f, "{:?}", s),
@@ -29,6 +44,8 @@ impl Display for Expression {
     }
 }
 
+debug_display!(Expr);
+
 /// An identifier.
 #[derive(Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
 pub struct Ident(pub String);
@@ -53,10 +70,15 @@ impl Display for Ident {
     }
 }
 
+debug_display!(Ident);
+
+#[derive(Debug, Clone, Eq, PartialEq)]
+pub struct StringLike(pub String);
+
 /// A sequence of expressions.
 #[derive(Clone, PartialEq)]
 pub struct Tuple {
-    pub items: Vec<Spanned<Expression>>,
+    pub items: Vec<Spanned<Expr>>,
 }
 
 impl Tuple {
@@ -64,7 +86,7 @@ impl Tuple {
         Tuple { items: vec![] }
     }
 
-    pub fn add(&mut self, item: Spanned<Expression>) {
+    pub fn add(&mut self, item: Spanned<Expr>) {
         self.items.push(item);
     }
 }
@@ -86,6 +108,8 @@ impl Display for Tuple {
     }
 }
 
+debug_display!(Tuple);
+
 /// A key-value collection of identifiers and associated expressions.
 #[derive(Clone, PartialEq)]
 pub struct Object {
@@ -97,7 +121,7 @@ impl Object {
         Object { pairs: vec![] }
     }
 
-    pub fn add(&mut self, key: Spanned<Ident>, value: Spanned<Expression>) {
+    pub fn add(&mut self, key: Spanned<Ident>, value: Spanned<Expr>) {
         self.pairs.push(Pair { key, value });
     }
 
@@ -127,11 +151,13 @@ impl Display for Object {
     }
 }
 
+debug_display!(Object);
+
 /// A key-value pair in an object.
 #[derive(Clone, PartialEq)]
 pub struct Pair {
     pub key: Spanned<Ident>,
-    pub value: Spanned<Expression>,
+    pub value: Spanned<Expr>,
 }
 
 impl Display for Pair {
@@ -140,57 +166,56 @@ impl Display for Pair {
     }
 }
 
-debug_display!(Ident);
-debug_display!(Expression);
-debug_display!(Tuple);
-debug_display!(Object);
 debug_display!(Pair);
 
-/// Kinds of expressions.
+pub trait ExprKind: Sized {
-pub trait ExpressionKind: Sized {
     /// The name of the expression in an `expected <name>` error.
     const NAME: &'static str;
 
     /// Create from expression.
-    fn from_expr(expr: Spanned<Expression>) -> ParseResult<Self>;
+    fn from_expr(expr: Spanned<Expr>) -> Result<Self, Error>;
 }
 
+impl<T> ExprKind for Spanned<T> where T: ExprKind {
+    const NAME: &'static str = T::NAME;
+
+    fn from_expr(expr: Spanned<Expr>) -> Result<Self, Error> {
+        let span = expr.span;
+        T::from_expr(expr).map(|v| Spanned { v, span })
+    }
+}
 /// Implements the expression kind trait for a type.
 macro_rules! kind {
-    ($type:ty, $name:expr, $($patterns:tt)*) => {
+    ($type:ty, $name:expr, $($p:pat => $r:expr),* $(,)?) => {
-        impl ExpressionKind for $type {
+        impl ExprKind for $type {
             const NAME: &'static str = $name;
 
-            fn from_expr(expr: Spanned<Expression>) -> ParseResult<Self> {
+            fn from_expr(expr: Spanned<Expr>) -> Result<Self, Error> {
                 #[allow(unreachable_patterns)]
                 Ok(match expr.v {
-                    $($patterns)*,
+                    $($p => $r),*,
-                    _ => error!("expected {}", Self::NAME),
+                    _ => return Err(
+                        err!("expected {}, found {}", Self::NAME, expr.v.name())
+                    ),
                 })
             }
         }
     };
 }
 
-kind!(Expression, "expression", e                          => e);
+kind!(Expr, "expression", e => e);
-kind!(Ident,      "identifier", Expression::Ident(ident)   => ident);
+kind!(Ident, "identifier", Expr::Ident(i) => i);
-kind!(String,     "string",     Expression::Str(string)    => string);
+kind!(String, "string", Expr::Str(s) => s);
-kind!(f64,        "number",     Expression::Number(num)    => num);
+kind!(f64, "number", Expr::Number(n) => n);
-kind!(bool,       "boolean",    Expression::Bool(boolean)  => boolean);
+kind!(bool, "boolean", Expr::Bool(b) => b);
-kind!(Size,       "size",       Expression::Size(size)     => size);
+kind!(Size, "size", Expr::Size(s) => s);
-kind!(Tuple,      "tuple",      Expression::Tuple(tuple)   => tuple);
+kind!(Tuple, "tuple", Expr::Tuple(t) => t);
-kind!(Object,     "object",     Expression::Object(object) => object);
+kind!(Object, "object", Expr::Object(o) => o);
-
+kind!(ScaleSize, "number or size",
-kind!(ScaleSize,  "number or size",
+    Expr::Size(size)    => ScaleSize::Absolute(size),
-    Expression::Size(size)    => ScaleSize::Absolute(size),
+    Expr::Number(scale) => ScaleSize::Scaled(scale as f32),
-    Expression::Number(scale) => ScaleSize::Scaled(scale as f32)
+);
+kind!(StringLike, "identifier or string",
+    Expr::Ident(Ident(s)) => StringLike(s),
+    Expr::Str(s) => StringLike(s),
 );
-
-impl<T> ExpressionKind for Spanned<T> where T: ExpressionKind {
-    const NAME: &'static str = T::NAME;
-
-    fn from_expr(expr: Spanned<Expression>) -> ParseResult<Spanned<T>> {
-        let span = expr.span;
-        T::from_expr(expr).map(|v| Spanned { v, span })
-    }
-}

src/syntax/func.rs

@@ -15,7 +15,7 @@ pub struct FuncArgs {
 
 #[derive(Debug, Clone, PartialEq)]
 pub enum Arg {
-    Pos(Spanned<Expression>),
+    Pos(Spanned<Expr>),
     Key(Pair),
 }
 
@@ -46,12 +46,12 @@ impl FuncArgs {
     }
 
     /// Add a positional argument.
-    pub fn add_pos(&mut self, item: Spanned<Expression>) {
+    pub fn add_pos(&mut self, item: Spanned<Expr>) {
         self.pos.add(item);
     }
 
     /// Add a keyword argument.
-    pub fn add_key(&mut self, key: Spanned<Ident>, value: Spanned<Expression>) {
+    pub fn add_key(&mut self, key: Spanned<Ident>, value: Spanned<Expr>) {
         self.key.add(key, value);
     }
 
@@ -92,7 +92,7 @@ impl FuncArgs {
     // }
 
     // /// Iterator over positional arguments.
-    // pub fn iter_pos(&mut self) -> std::vec::IntoIter<Spanned<Expression>> {
+    // pub fn iter_pos(&mut self) -> std::vec::IntoIter<Spanned<Expr>> {
     //     let tuple = std::mem::replace(&mut self.positional, Tuple::new());
     //     tuple.items.into_iter()
     // }

src/syntax/mod.rs

@@ -23,22 +23,6 @@ pub mod prelude {
 }
 
 
-pub struct Parsed<T> {
-    pub output: T,
-    pub errors: SpanVec<Error>,
-    pub decorations: SpanVec<Decoration>,
-}
-
-impl<T> Parsed<T> {
-    pub fn map<F, U>(self, f: F) -> Parsed<U> where F: FnOnce(T) -> U {
-        Parsed {
-            output: f(self.output),
-            errors: self.errors,
-            decorations: self.decorations,
-        }
-    }
-}
-
 #[async_trait::async_trait(?Send)]
 pub trait Model: Debug + ModelBounds {
     async fn layout<'a>(
@@ -110,7 +94,7 @@ impl SyntaxModel {
 
 #[async_trait::async_trait(?Send)]
 impl Model for SyntaxModel {
-    async fn layout<'a>(&'a self, ctx: LayoutContext<'_, '_>) -> Layouted<Commands<'a>> {
+    async fn layout<'a>(&'a self, _: LayoutContext<'_, '_>) -> Layouted<Commands<'a>> {
         Layouted {
             output: vec![Command::LayoutSyntaxModel(self)],
             errors: vec![],
@@ -153,7 +137,8 @@ impl PartialEq for Node {
     }
 }
 
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize)]
+#[serde(rename_all = "camelCase")]
 pub enum Decoration {
     ValidFuncName,
     InvalidFuncName,

src/syntax/parsing.rs

@@ -3,10 +3,6 @@ use super::*;
 use Token::*;
 
 
-pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed<SyntaxModel> {
-    Parser::new(start, src, ctx).parse()
-}
-
 /// The context for parsing.
 #[derive(Debug, Copy, Clone)]
 pub struct ParseContext<'a> {
@@ -14,185 +10,169 @@ pub struct ParseContext<'a> {
     pub scope: &'a Scope,
 }
 
-struct Parser<'s> {
+pub struct Parsed<T> {
-    src: &'s str,
+    pub output: T,
-    ctx: ParseContext<'s>,
+    pub errors: SpanVec<Error>,
-    tokens: Tokens<'s>,
+    pub decorations: SpanVec<Decoration>,
-    peeked: Option<Option<Spanned<Token<'s>>>>,
-    position: Position,
-    last_position: Position,
-    errors: SpanVec<Error>,
-    decorations: SpanVec<Decoration>,
 }
 
-impl<'s> Parser<'s> {
+impl<T> Parsed<T> {
-    fn new(start: Position, src: &'s str, ctx: ParseContext<'s>) -> Parser<'s> {
+    pub fn map<F, U>(self, f: F) -> Parsed<U> where F: FnOnce(T) -> U {
-        Parser {
+        Parsed {
-            src,
+            output: f(self.output),
+            errors: self.errors,
+            decorations: self.decorations,
+        }
+    }
+}
+
+pub fn parse(start: Position, src: &str, ctx: ParseContext) -> Parsed<SyntaxModel> {
+    let mut model = SyntaxModel::new();
+    let mut errors = Vec::new();
+    let mut decorations = Vec::new();
+
+    let mut tokens = Tokens::new(start, src, TokenizationMode::Body);
+
+    while let Some(token) = tokens.next() {
+        let span = token.span;
+
+        let node = match token.v {
+            Space(newlines) => if newlines >= 2 {
+                Node::Newline
+            } else {
+                Node::Space
+            },
+
+            Function { header, body, terminated } => {
+                let parsed: Parsed<Node> = FuncParser::new(header, body, ctx).parse();
+
+                errors.extend(offset_spans(parsed.errors, span.start));
+                decorations.extend(offset_spans(parsed.decorations, span.start));
+
+                if !terminated {
+                    errors.push(err!(Span::at(span.end); "expected closing bracket"));
+                }
+
+                parsed.output
+            }
+
+            Star       => Node::ToggleBolder,
+            Underscore => Node::ToggleItalic,
+            Backtick   => Node::ToggleMonospace,
+            Text(text) => Node::Text(text.to_owned()),
+
+            LineComment(_) | BlockComment(_) => continue,
+
+            other => {
+                errors.push(err!(span; "unexpected {}", name(other)));
+                continue;
+            }
+        };
+
+        model.add(Spanned { v: node, span: token.span });
+    }
+
+    Parsed { output: model, errors, decorations }
+}
+
+struct FuncParser<'s> {
+    ctx: ParseContext<'s>,
+    errors: SpanVec<Error>,
+    decorations: SpanVec<Decoration>,
+    tokens: Tokens<'s>,
+    peeked: Option<Option<Spanned<Token<'s>>>>,
+    body: Option<(Position, &'s str)>,
+}
+
+impl<'s> FuncParser<'s> {
+    fn new(
+        header: &'s str,
+        body: Option<(Position, &'s str)>,
+        ctx: ParseContext<'s>
+    ) -> FuncParser<'s> {
+        FuncParser {
             ctx,
-            tokens: tokenize(start, src),
-            peeked: None,
-            position: Position::ZERO,
-            last_position: Position::ZERO,
             errors: vec![],
             decorations: vec![],
+            tokens: Tokens::new(Position::new(0, 1), header, TokenizationMode::Header),
+            peeked: None,
+            body,
         }
     }
 
-    /// The main parsing entrypoint.
+    fn parse(mut self) -> Parsed<Node> {
-    fn parse(mut self) -> Parsed<SyntaxModel> {
+        let parsed = if let Some(header) = self.parse_func_header() {
-        let mut model = SyntaxModel::new();
+            let name = header.name.v.as_str();
-
+            let (parser, deco) = match self.ctx.scope.get_parser(name) {
-        while let Some(token) = self.eat() {
+                Ok(parser) => (parser, Decoration::ValidFuncName),
-            let mut span = token.span;
+                Err(parser) => {
-            let node = match token.v {
+                    self.errors.push(err!(header.name.span; "unknown function"));
-                LineComment(_) | BlockComment(_) => None,
+                    (parser, Decoration::InvalidFuncName)
-                Whitespace(newlines) => Some(if newlines >= 2 {
-                    Node::Newline
-                } else {
-                    Node::Space
-                }),
-
-                LeftBracket => self.parse_func().map(|spanned| {
-                    span = spanned.span;
-                    spanned.v
-                }),
-
-                Star       => Some(Node::ToggleBolder),
-                Underscore => Some(Node::ToggleItalic),
-                Backtick   => Some(Node::ToggleMonospace),
-                Text(text) => Some(Node::Text(text.to_owned())),
-
-                _ => {
-                    self.unexpected(token);
-                    None
                 }
             };
 
-            if let Some(v) = node {
+            self.decorations.push(Spanned::new(deco, header.name.span));
-                model.add(Spanned { v, span });
+
-            }
+            parser(header, self.body, self.ctx)
-        }
+        } else {
+            let default = FuncHeader {
+                name: Spanned::new(Ident("".to_string()), Span::ZERO),
+                args: FuncArgs::new(),
+            };
+
+            // Use the fallback function such that the body is still rendered
+            // even if the header is completely unparsable.
+            self.ctx.scope.get_fallback_parser()(default, self.body, self.ctx)
+        };
+
+        self.errors.extend(parsed.errors);
+        self.decorations.extend(parsed.decorations);
 
         Parsed {
-            output: model,
+            output: Node::Model(parsed.output),
             errors: self.errors,
             decorations: self.decorations,
         }
     }
 
-    /// Parses a function including header and body with the cursor starting
-    /// right behind the first opening bracket.
-    fn parse_func(&mut self) -> Option<Spanned<Node>> {
-        let start = self.last_pos();
-
-        let header = self.parse_func_header();
-        self.eat_until(|t| t == RightBracket, false);
-
-        if self.eat().map(Spanned::value) != Some(RightBracket) {
-            self.expected_at("closing bracket", self.pos());
-        }
-
-        let body = if self.peekv() == Some(LeftBracket) {
-            self.eat();
-
-            let start_index = self.tokens.index();
-            let start_position = self.tokens.pos();
-
-            let found = self.tokens.move_to_closing_bracket();
-
-            let end_index = self.tokens.index();
-            let end_position = self.tokens.pos();
-
-            let body = &self.src[start_index .. end_index];
-
-            self.position = end_position;
-
-            if found {
-                let next = self.eat().map(Spanned::value);
-                debug_assert_eq!(next, Some(RightBracket));
-            } else {
-                self.expected_at("closing bracket", self.pos());
-            }
-
-            Some(Spanned::new(body, Span::new(start_position, end_position)))
-        } else {
-            None
-        };
-
-        let header = header?;
-        let (parser, decoration) = match self.ctx.scope.get_parser(header.name.v.as_str()) {
-            Ok(parser) => (parser, Decoration::ValidFuncName),
-            Err(parser) => {
-                let error = Error::new(format!("unknown function: `{}`", header.name.v));
-                self.errors.push(Spanned::new(error, header.name.span));
-                (parser, Decoration::InvalidFuncName)
-            }
-        };
-
-        self.decorations.push(Spanned::new(decoration, header.name.span));
-
-        let parsed = parser(header, body, self.ctx);
-        self.errors.extend(offset_spans(parsed.errors, start));
-        self.decorations.extend(offset_spans(parsed.decorations, start));
-
-        let node = Node::Model(parsed.output);
-
-        let end = self.pos();
-        let span = Span { start, end };
-
-        Some(Spanned { v: node, span })
-    }
-
-    /// Parses a function header including the closing bracket.
     fn parse_func_header(&mut self) -> Option<FuncHeader> {
+        let start = self.pos();
         self.skip_whitespace();
-        let name = self.parse_func_name()?;
+
+        let name = match self.eat() {
+            Some(Spanned { v: ExprIdent(ident), span }) => {
+                Spanned { v: Ident(ident.to_string()), span }
+            }
+            other => {
+                self.expected_found_or_at("identifier", other, start);
+                return None;
+            }
+        };
 
         self.skip_whitespace();
-        let args = match self.peek() {
+        let args = match self.eat().map(Spanned::value) {
-            Some(Spanned { v: Colon, .. }) => {
+            Some(Colon) => self.parse_func_args(),
-                self.eat();
+            Some(_) => {
-                self.parse_func_args()
+                self.expected_at("colon", name.span.end);
-            }
-            Some(Spanned { v: RightBracket, .. }) => FuncArgs::new(),
-            other => {
-                self.expected_at("colon or closing bracket", name.span.end);
                 FuncArgs::new()
             }
+            None => FuncArgs::new(),
         };
 
         Some(FuncHeader { name, args })
     }
 
-    /// Parses the function name if is the next token. Otherwise, it adds an
-    /// error and returns `None`.
-    fn parse_func_name(&mut self) -> Option<Spanned<Ident>> {
-        match self.peek() {
-            Some(Spanned { v: ExprIdent(ident), span }) => {
-                self.eat();
-                return Some(Spanned { v: Ident(ident.to_string()), span });
-            }
-            other => self.expected_found_or_at("identifier", other, self.pos()),
-        }
-
-        None
-    }
-
-    /// Parses the function arguments and stops right before the final closing
-    /// bracket.
     fn parse_func_args(&mut self) -> FuncArgs {
         let mut args = FuncArgs::new();
 
-        loop {
+        self.skip_whitespace();
-            self.skip_whitespace();
+        while self.peek().is_some() {
-            match self.peekv() {
+            match self.parse_arg() {
-                Some(RightBracket) | None => break,
+                Some(arg) => args.add(arg),
-                _ => match self.parse_arg() {
+                None => {}
-                    Some(arg) => args.add(arg),
-                    None => {}
-                }
             }
+
+            self.skip_whitespace();
         }
 
         args
@@ -221,7 +201,7 @@ impl<'s> Parser<'s> {
                     })
                 })
             } else {
-                Some(Arg::Pos(Spanned::new(Expression::Ident(ident), span)))
+                Some(Arg::Pos(Spanned::new(Expr::Ident(ident), span)))
             }
         } else {
             self.parse_expr().map(|expr| Arg::Pos(expr))
@@ -230,7 +210,6 @@ impl<'s> Parser<'s> {
         if let Some(arg) = &arg {
             self.skip_whitespace();
             match self.peekv() {
-                Some(RightBracket) => {}
                 Some(Comma) => { self.eat(); }
                 Some(_) => self.expected_at("comma", arg.span().end),
                 _ => {}
@@ -244,19 +223,27 @@ impl<'s> Parser<'s> {
     }
 
     /// Parse a atomic or compound (tuple / object) expression.
-    fn parse_expr(&mut self) -> Option<Spanned<Expression>> {
+    fn parse_expr(&mut self) -> Option<Spanned<Expr>> {
         let first = self.peek()?;
-        let mut expr = |v| {
+        let spanned = |v| Spanned { v, span: first.span };
-            self.eat();
-            Spanned { v, span: first.span }
-        };
 
         Some(match first.v {
-            ExprIdent(i) => expr(Expression::Ident(Ident(i.to_string()))),
+            ExprIdent(i) => {
-            ExprStr(s) => expr(Expression::Str(s.to_string())),
+                self.eat();
-            ExprNumber(n) => expr(Expression::Number(n)),
+                spanned(Expr::Ident(Ident(i.to_string())))
-            ExprSize(s) => expr(Expression::Size(s)),
+            }
-            ExprBool(b) => expr(Expression::Bool(b)),
+            ExprStr { string, terminated } => {
+                if !terminated {
+                    self.expected_at("quote", first.span.end);
+                }
+
+                self.eat();
+                spanned(Expr::Str(string.to_string()))
+            }
+            ExprNumber(n) => { self.eat(); spanned(Expr::Number(n)) }
+            ExprSize(s) => { self.eat(); spanned(Expr::Size(s)) }
+            ExprBool(b) => { self.eat(); spanned(Expr::Bool(b)) }
+
             LeftParen => self.parse_tuple(),
             LeftBrace => self.parse_object(),
             _ => return None,
@@ -264,56 +251,48 @@ impl<'s> Parser<'s> {
     }
 
     /// Parse a tuple expression.
-    fn parse_tuple(&mut self) -> Spanned<Expression> {
+    fn parse_tuple(&mut self) -> Spanned<Expr> {
         let start = self.pos();
 
         // TODO: Do the thing.
-        self.eat_until(|t| matches!(t, RightParen | RightBracket), false);
+        self.eat_until(|t| t == RightParen, true);
-        if self.peekv() == Some(RightParen) {
-            self.eat();
-        }
 
         let end = self.pos();
         let span = Span { start, end };
 
-        Spanned { v: Expression::Tuple(Tuple::new()), span }
+        Spanned { v: Expr::Tuple(Tuple::new()), span }
     }
 
     /// Parse an object expression.
-    fn parse_object(&mut self) -> Spanned<Expression> {
+    fn parse_object(&mut self) -> Spanned<Expr> {
         let start = self.pos();
 
         // TODO: Do the thing.
-        self.eat_until(|t| matches!(t, RightBrace | RightBracket), false);
+        self.eat_until(|t| t == RightBrace, true);
-        if self.peekv() == Some(RightBrace) {
-            self.eat();
-        }
 
         let end = self.pos();
         let span = Span { start, end };
 
-        Spanned { v: Expression::Object(Object::new()), span }
+        Spanned { v: Expr::Object(Object::new()), span }
     }
 
     /// Skip all whitespace/comment tokens.
     fn skip_whitespace(&mut self) {
         self.eat_until(|t|
-            !matches!(t, Whitespace(_) | LineComment(_) | BlockComment(_)), false)
+            !matches!(t, Space(_) | LineComment(_) | BlockComment(_)), false)
-    }
-
-    /// Add an error about an `thing` which was expected but not found at the
-    /// given position.
-    fn expected_at(&mut self, thing: &str, pos: Position) {
-        let error = Error::new(format!("expected {}", thing));
-        self.errors.push(Spanned::new(error, Span::at(pos)));
     }
 
     /// Add an error about an expected `thing` which was not found, showing
     /// what was found instead.
     fn expected_found(&mut self, thing: &str, found: Spanned<Token>) {
-        let message = format!("expected {}, found {}", thing, name(found.v));
+        self.errors.push(err!(found.span;
-        let error = Error::new(message);
+            "expected {}, found {}", thing, name(found.v)));
-        self.errors.push(Spanned::new(error, found.span));
+    }
+
+    /// Add an error about an `thing` which was expected but not found at the
+    /// given position.
+    fn expected_at(&mut self, thing: &str, pos: Position) {
+        self.errors.push(err!(Span::at(pos); "expected {}", thing));
     }
 
     /// Add a found-error if `found` is some and a positional error, otherwise.
@@ -329,12 +308,6 @@ impl<'s> Parser<'s> {
         }
     }
 
-    /// Add an error about an unexpected token `found`.
-    fn unexpected(&mut self, found: Spanned<Token>) {
-        let error = Error::new(format!("unexpected {}", name(found.v)));
-        self.errors.push(Spanned::new(error, found.span));
-    }
-
     /// Consume tokens until the function returns true and only consume the last
     /// token if instructed to.
     fn eat_until<F>(&mut self, mut f: F, eat_match: bool)
@@ -351,19 +324,10 @@ impl<'s> Parser<'s> {
         }
     }
 
-    /// Consume and return the next token, update positions and colorize the
+    /// Consume and return the next token.
-    /// token. All colorable tokens are per default colorized here, to override
-    /// a colorization use `Colorization::replace_last`.
     fn eat(&mut self) -> Option<Spanned<Token<'s>>> {
-        let token = self.peeked.take()
+        self.peeked.take()
-            .unwrap_or_else(|| self.tokens.next());
+            .unwrap_or_else(|| self.tokens.next())
-
-        if let Some(token) = token {
-            self.last_position = self.position;
-            self.position = token.span.end;
-        }
-
-        token
     }
 
     /// Peek at the next token without consuming it.
@@ -379,23 +343,18 @@ impl<'s> Parser<'s> {
     /// The position at the end of the last eat token / start of the peekable
     /// token.
     fn pos(&self) -> Position {
-        self.position
+        self.peeked.flatten()
-    }
+            .map(|s| s.span.start)
-
+            .unwrap_or_else(|| self.tokens.pos())
-    /// The position at the start of the last eaten token.
-    fn last_pos(&self) -> Position {
-        self.last_position
     }
 }
 
-/// The name of a token in an `expected <...>` error.
+/// The name of a token in an `(un)expected <...>` error.
 fn name(token: Token) -> &'static str {
     match token {
-        Whitespace(_) => "whitespace",
+        Space(_)      => "space",
         LineComment(_) | BlockComment(_) => "comment",
-        StarSlash     => "end of block comment",
+        Function { .. } => "function",
-        LeftBracket   => "opening bracket",
-        RightBracket  => "closing bracket",
         LeftParen     => "opening paren",
         RightParen    => "closing paren",
         LeftBrace     => "opening brace",
@@ -404,13 +363,16 @@ fn name(token: Token) -> &'static str {
         Comma         => "comma",
         Equals        => "equals sign",
         ExprIdent(_)  => "identifier",
-        ExprStr(_)    => "string",
+        ExprStr { .. }    => "string",
         ExprNumber(_) => "number",
         ExprSize(_)   => "size",
-        ExprBool(_)   => "bool",
+        ExprBool(_)   => "boolean",
         Star          => "star",
         Underscore    => "underscore",
         Backtick      => "backtick",
         Text(_)       => "invalid identifier",
+        Invalid("]")  => "closing bracket",
+        Invalid("*/") => "end of block comment",
+        Invalid(_)    => "invalid token",
     }
 }

src/syntax/span.rs

@@ -1,7 +1,7 @@
 //! Spans map elements to the part of source code they originate from.
 
 use std::fmt::{self, Debug, Display, Formatter};
-use std::ops::{Add, AddAssign};
+use std::ops::{Add, Sub};
 use serde::Serialize;
 
 
@@ -21,12 +21,13 @@ impl<T> Spanned<T> {
         self.v
     }
 
-    pub fn map<F, V>(self, f: F) -> Spanned<V> where F: FnOnce(T) -> V {
+    pub fn map<V, F>(self, f: F) -> Spanned<V> where F: FnOnce(T) -> V {
         Spanned { v: f(self.v), span: self.span }
     }
 
-    pub fn map_v<V>(&self, new_v: V) -> Spanned<V> {
+    pub fn map_span<F>(mut self, f: F) -> Spanned<T> where F: FnOnce(Span) -> Span {
-        Spanned { v: new_v, span: self.span }
+        self.span = f(self.span);
+        self
     }
 }
 
@@ -74,6 +75,13 @@ impl Span {
     pub fn expand(&mut self, other: Span) {
         *self = Span::merge(*self, other)
     }
+
+    pub fn offset(self, start: Position) -> Span {
+        Span {
+            start: start + self.start,
+            end: start + self.end,
+        }
+    }
 }
 
 impl Display for Span {
@@ -119,9 +127,21 @@ impl Add for Position {
     }
 }
 
-impl AddAssign for Position {
+impl Sub for Position {
-    fn add_assign(&mut self, other: Position) {
+    type Output = Position;
-        *self = *self + other;
+
+    fn sub(self, rhs: Position) -> Position {
+        if self.line == rhs.line {
+            Position {
+                line: 0,
+                column: self.column - rhs.column
+            }
+        } else {
+            Position {
+                line: self.line - rhs.line,
+                column: self.column,
+            }
+        }
     }
 }
 
@@ -136,14 +156,6 @@ debug_display!(Position);
 /// A vector of spanned things.
 pub type SpanVec<T> = Vec<Spanned<T>>;
 
-pub fn offset_spans<T>(
+pub fn offset_spans<T>(vec: SpanVec<T>, start: Position) -> impl Iterator<Item=Spanned<T>> {
-    vec: SpanVec<T>,
+    vec.into_iter().map(move |s| s.map_span(|span| span.offset(start)))
-    start: Position,
-) -> impl Iterator<Item=Spanned<T>> {
-    vec.into_iter()
-        .map(move |mut spanned| {
-            spanned.span.start += start;
-            spanned.span.end += start;
-            spanned
-        })
 }

src/syntax/tokens.rs

@@ -4,7 +4,7 @@ use unicode_xid::UnicodeXID;
 
 use super::*;
 use Token::*;
-use State::*;
+use TokenizationMode::*;
 
 
 /// A minimal semantic entity of source code.
@@ -12,20 +12,20 @@ use State::*;
 pub enum Token<'s> {
     /// One or more whitespace characters. The contained `usize` denotes the
     /// number of newlines that were contained in the whitespace.
-    Whitespace(usize),
+    Space(usize),
 
     /// A line comment with inner string contents `//<&'s str>\n`.
     LineComment(&'s str),
     /// A block comment with inner string contents `/*<&'s str>*/`. The comment
     /// can contain nested block comments.
     BlockComment(&'s str),
-    /// An erroneous `*/` without an opening block comment.
-    StarSlash,
 
-    /// A left bracket: `[`.
+    /// A function invocation `[<header>][<body>]`.
-    LeftBracket,
+    Function {
-    /// A right bracket: `]`.
+        header: &'s str,
-    RightBracket,
+        body: Option<(Position, &'s str)>,
+        terminated: bool,
+    },
 
     /// A left parenthesis in a function header: `(`.
     LeftParen,
@@ -46,7 +46,7 @@ pub enum Token<'s> {
     /// An identifier in a function header: `center`.
     ExprIdent(&'s str),
     /// A quoted string in a function header: `"..."`.
-    ExprStr(&'s str),
+    ExprStr { string: &'s str, terminated: bool },
     /// A number in a function header: `3.14`.
     ExprNumber(f64),
     /// A size in a function header: `12pt`.
@@ -63,36 +63,31 @@ pub enum Token<'s> {
 
     /// Any other consecutive string.
     Text(&'s str),
-}
 
-/// Decomposes text into a sequence of semantic tokens.
+    /// Things that are not valid in the context they appeared in.
-pub fn tokenize(start: Position, src: &str) -> Tokens {
+    Invalid(&'s str),
-    Tokens::new(start, src)
 }
 
 /// An iterator over the tokens of a string of source code.
 pub struct Tokens<'s> {
     src: &'s str,
-    state: State,
+    mode: TokenizationMode,
-    stack: Vec<(State, Position)>,
     iter: Peekable<Chars<'s>>,
     position: Position,
     index: usize,
 }
 
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
-enum State {
+pub enum TokenizationMode {
     Header,
-    StartBody,
     Body,
 }
 
 impl<'s> Tokens<'s> {
-    pub fn new(start: Position, src: &'s str) -> Tokens<'s> {
+    pub fn new(start: Position, src: &'s str, mode: TokenizationMode) -> Tokens<'s> {
         Tokens {
             src,
-            state: State::Body,
+            mode,
-            stack: vec![],
             iter: src.chars().peekable(),
             position: start,
             index: 0,
@@ -110,35 +105,6 @@ impl<'s> Tokens<'s> {
     pub fn pos(&self) -> Position {
         self.position
     }
-
-    /// Move through the string until an unbalanced closing bracket is found
-    /// without tokenizing the contents.
-    ///
-    /// Returns whether a closing bracket was found or the end of the string was
-    /// reached.
-    pub fn move_to_closing_bracket(&mut self) -> bool {
-        let mut escaped = false;
-        let mut depth = 0;
-
-        self.read_string_until(|n| {
-            match n {
-                '[' if !escaped => depth += 1,
-                ']' if !escaped => {
-                    if depth == 0 {
-                        return true;
-                    } else {
-                        depth -= 1;
-                    }
-                }
-                '\\' => escaped = !escaped,
-                _ => escaped = false,
-            }
-
-            false
-        }, false, 0, 0);
-
-        self.peek() == Some(']')
-    }
 }
 
 impl<'s> Iterator for Tokens<'s> {
@@ -153,55 +119,31 @@ impl<'s> Iterator for Tokens<'s> {
             // Comments.
             '/' if self.peek() == Some('/') => self.parse_line_comment(),
             '/' if self.peek() == Some('*') => self.parse_block_comment(),
-            '*' if self.peek() == Some('/') => { self.eat(); StarSlash }
+            '*' if self.peek() == Some('/') => { self.eat(); Invalid("*/") }
 
             // Whitespace.
             c if c.is_whitespace() => self.parse_whitespace(start),
 
             // Functions.
-            '[' => {
+            '[' => self.parse_function(start),
-                match self.state {
+            ']' => Invalid("]"),
-                    Header | Body => {
-                        self.stack.push((self.state, start));
-                        self.position = Position::new(0, '['.len_utf8());
-                        self.state = Header;
-                    }
-                    StartBody => self.state = Body,
-                }
-
-                LeftBracket
-            }
-            ']' => {
-                if self.state == Header && self.peek() == Some('[') {
-                    self.state = StartBody;
-                } else {
-                    if let Some((state, pos)) = self.stack.pop() {
-                        self.state = state;
-                        self.position = pos + self.position;
-                    } else {
-                        self.state = Body;
-                    }
-                }
-
-                RightBracket
-            }
 
             // Syntactic elements in function headers.
-            '(' if self.state == Header => LeftParen,
+            '(' if self.mode == Header => LeftParen,
-            ')' if self.state == Header => RightParen,
+            ')' if self.mode == Header => RightParen,
-            '{' if self.state == Header => LeftBrace,
+            '{' if self.mode == Header => LeftBrace,
-            '}' if self.state == Header => RightBrace,
+            '}' if self.mode == Header => RightBrace,
-            ':' if self.state == Header => Colon,
+            ':' if self.mode == Header => Colon,
-            ',' if self.state == Header => Comma,
+            ',' if self.mode == Header => Comma,
-            '=' if self.state == Header => Equals,
+            '=' if self.mode == Header => Equals,
 
             // String values.
-            '"' if self.state == Header => self.parse_string(),
+            '"' if self.mode == Header => self.parse_string(),
 
             // Style toggles.
-            '*' if self.state == Body => Star,
+            '*' if self.mode == Body => Star,
-            '_' if self.state == Body => Underscore,
+            '_' if self.mode == Body => Underscore,
-            '`' if self.state == Body => Backtick,
+            '`' if self.mode == Body => Backtick,
 
             // An escaped thing.
             '\\' => self.parse_escaped(),
@@ -215,9 +157,9 @@ impl<'s> Iterator for Tokens<'s> {
                         ',' | '"' | '/' => true,
                         _ => false,
                     }
-                }, false, -(c.len_utf8() as isize), 0);
+                }, false, -(c.len_utf8() as isize), 0).0;
 
-                if self.state == Header {
+                if self.mode == Header {
                     self.parse_expr(text)
                 } else {
                     Text(text)
@@ -234,7 +176,7 @@ impl<'s> Iterator for Tokens<'s> {
 
 impl<'s> Tokens<'s> {
     fn parse_line_comment(&mut self) -> Token<'s> {
-        LineComment(self.read_string_until(is_newline_char, false, 1, 0))
+        LineComment(self.read_string_until(is_newline_char, false, 1, 0).0)
     }
 
     fn parse_block_comment(&mut self) -> Token<'s> {
@@ -262,19 +204,60 @@ impl<'s> Tokens<'s> {
             }
 
             false
-        }, true, 0, -2))
+        }, true, 0, -2).0)
     }
 
     fn parse_whitespace(&mut self, start: Position) -> Token<'s> {
         self.read_string_until(|n| !n.is_whitespace(), false, 0, 0);
         let end = self.pos();
 
-        Whitespace(end.line - start.line)
+        Space(end.line - start.line)
+    }
+
+    fn parse_function(&mut self, start: Position) -> Token<'s> {
+        let (header, terminated) = self.read_function_part();
+        self.eat();
+
+        if self.peek() != Some('[') {
+            return Function { header, body: None, terminated };
+        }
+
+        self.eat();
+
+        let offset = self.pos() - start;
+        let (body, terminated) = self.read_function_part();
+        self.eat();
+
+        Function { header, body: Some((offset, body)), terminated }
+    }
+
+    fn read_function_part(&mut self) -> (&'s str, bool) {
+        let mut escaped = false;
+        let mut in_string = false;
+        let mut depth = 0;
+
+        self.read_string_until(|n| {
+            match n {
+                '"' if !escaped => in_string = !in_string,
+                '[' if !escaped && !in_string => depth += 1,
+                ']' if !escaped && !in_string => {
+                    if depth == 0 {
+                        return true;
+                    } else {
+                        depth -= 1;
+                    }
+                }
+                '\\' => escaped = !escaped,
+                _ => escaped = false,
+            }
+
+            false
+        }, false, 0, 0)
     }
 
     fn parse_string(&mut self) -> Token<'s> {
         let mut escaped = false;
-        ExprStr(self.read_string_until(|n| {
+        let (string, terminated) = self.read_string_until(|n| {
             match n {
                 '"' if !escaped => return true,
                 '\\' => escaped = !escaped,
@@ -282,7 +265,8 @@ impl<'s> Tokens<'s> {
             }
 
             false
-        }, true, 0, -1))
+        }, true, 0, -1);
+        ExprStr { string, terminated }
     }
 
     fn parse_escaped(&mut self) -> Token<'s> {
@@ -294,7 +278,7 @@ impl<'s> Tokens<'s> {
         }
 
         let c = self.peek().unwrap_or('n');
-        if self.state == Body && is_escapable(c) {
+        if self.mode == Body && is_escapable(c) {
             let index = self.index();
             self.eat();
             Text(&self.src[index .. index + c.len_utf8()])
@@ -315,7 +299,7 @@ impl<'s> Tokens<'s> {
         } else if is_identifier(text) {
             ExprIdent(text)
         } else {
-            Text(text)
+            Invalid(text)
         }
     }
 
@@ -325,7 +309,7 @@ impl<'s> Tokens<'s> {
         eat_match: bool,
         offset_start: isize,
         offset_end: isize,
-    ) -> &'s str where F: FnMut(char) -> bool {
+    ) -> (&'s str, bool) where F: FnMut(char) -> bool {
         let start = ((self.index() as isize) + offset_start) as usize;
         let mut matched = false;
 
@@ -346,7 +330,7 @@ impl<'s> Tokens<'s> {
             end = ((end as isize) + offset_end) as usize;
         }
 
-        &self.src[start .. end]
+        (&self.src[start .. end], matched)
     }
 
     fn eat(&mut self) -> Option<char> {

tests/layouter/test.typ

@@ -1 +0,0 @@
-[box][hi]

tests/parser/tokens.rs

@@ -1,80 +1,77 @@
 // Whitespace.
 t ""             => []
-t " "            => [W(0)]
+t " "            => [S(0)]
-t "    "         => [W(0)]
+t "    "         => [S(0)]
-t "\t"           => [W(0)]
+t "\t"           => [S(0)]
-t "  \t"         => [W(0)]
+t "  \t"         => [S(0)]
-t "\n"           => [W(1)]
+t "\n"           => [S(1)]
-t "\n "          => [W(1)]
+t "\n "          => [S(1)]
-t "  \n"         => [W(1)]
+t "  \n"         => [S(1)]
-t "  \n   "      => [W(1)]
+t "  \n   "      => [S(1)]
-t "  \n\t \n  "  => [W(2)]
+t "  \n\t \n  "  => [S(2)]
-t "\r\n"         => [W(1)]
+t "\r\n"         => [S(1)]
-t " \r\r\n \x0D" => [W(3)]
+t " \r\r\n \x0D" => [S(3)]
-t "\n\r"         => [W(2)]
+t "\n\r"         => [S(2)]
 
 // Comments.
-t "a // bc\n "        => [T("a"), W(0), LC(" bc"), W(1)]
+t "a // bc\n "        => [T("a"), S(0), LC(" bc"),  S(1)]
-t "a //a//b\n "       => [T("a"), W(0), LC("a//b"), W(1)]
+t "a //a//b\n "       => [T("a"), S(0), LC("a//b"), S(1)]
-t "a //a//b\r\n"      => [T("a"), W(0), LC("a//b"), W(1)]
+t "a //a//b\r\n"      => [T("a"), S(0), LC("a//b"), S(1)]
-t "a //a//b\n\nhello" => [T("a"), W(0), LC("a//b"), W(2), T("hello")]
+t "a //a//b\n\nhello" => [T("a"), S(0), LC("a//b"), S(2), T("hello")]
 t "/**/"              => [BC("")]
-t "_/*_/*a*/*/"       => [U, BC("_/*a*/")]
+t "_/*_/*a*/*/"       => [Underscore, BC("_/*a*/")]
 t "/*/*/"             => [BC("/*/")]
-t "abc*/"             => [T("abc"), SS]
+t "abc*/"             => [T("abc"), Invalid("*/")]
 
 // Header only tokens.
-t "["                   => [LB]
+th "["                 => [Func("", None, false)]
-t "]"                   => [RB]
+th "]"                 => [Invalid("]")]
-t "[(){}:=,]"           => [LB, LP, RP, LBR, RBR, CL, EQ, CM, RB]
+th "(){}:=,"           => [LP, RP, LB, RB, Colon, Equals, Comma]
-t "[a:b]"               => [LB, ID("a"), CL, ID("b"), RB]
+th "a:b"               => [Id("a"), Colon, Id("b")]
-t "[🌓, 🌍,]"           => [LB, T("🌓"), CM, W(0), T("🌍"), CM, RB]
+th "="                 => [Equals]
-t "[=]"                 => [LB, EQ, RB]
+th ","                 => [Comma]
-t "[,]"                 => [LB, CM, RB]
+th r#""hello\"world""# => [Str(r#"hello\"world"#)]
-t "a: b"                => [T("a"), T(":"), W(0), T("b")]
+th r#""hi", 12pt"#     => [Str("hi"), Comma, S(0), Size(12.0)]
-t "c=d, "               => [T("c"), T("=d"), T(","), W(0)]
+th "\"hi\""            => [T("\"hi"), T("\"")]
-t r#"["hello\"world"]"# => [LB, STR(r#"hello\"world"#), RB]
+th "a: true, x=1"      => [Id("a"), Colon, S(0), Bool(true), Comma, S(0),
-t r#"["hi", 12pt]"#     => [LB, STR("hi"), CM, W(0), SIZE(Size::pt(12.0)), RB]
+                           Id("x"), Equals, Num(1.0)]
-t "\"hi\""              => [T("\"hi"), T("\"")]
+th "120%"              => [Num(1.2)]
-t "[a: true, x=1]"      => [LB, ID("a"), CL, W(0), BOOL(true), CM, W(0),
+th "🌓, 🌍,"           => [T("🌓"), Comma, S(0), T("🌍"), Comma]
-                          ID("x"), EQ, NUM(1.0), RB]
+tb "a: b"              => [T("a"), T(":"), S(0), T("b")]
-t "[120%]"              => [LB, NUM(1.2), RB]
+tb "c=d, "             => [T("c"), T("=d"), T(","), S(0)]
 
 // Body only tokens.
-t "_*`"           => [U, S, B]
+tb "_*`"           => [Underscore, Star, Backtick]
-t "[func]*bold*"  => [LB, ID("func"), RB, S, T("bold"), S]
+tb "[func]*bold*"  => [Func("func", None, true), Star, T("bold"), Star]
-t "[_*`]"         => [LB, T("_"), T("*"), T("`"), RB]
+tb "hi_you_ there" => [T("hi"), Underscore, T("you"), Underscore, S(0), T("there")]
-t "hi_you_ there" => [T("hi"), U, T("you"), U, W(0), T("there")]
+th "_*`"           => [Invalid("_"), Invalid("*"), Invalid("`")]
 
 // Nested functions.
-t "[f: [=][*]]"    => [LB, ID("f"), CL, W(0), LB, EQ, RB, LB, S, RB, RB]
+tb "[f: [=][*]]"    => [Func("f: [=][*]", None, true)]
-t "[_][[,],],"     => [LB, T("_"), RB, LB, LB, CM, RB, T(","), RB, T(",")]
+tb "[_][[,],],"     => [Func("_", Some("[,],"), true), T(",")]
-t "[=][=][=]"      => [LB, EQ, RB, LB, T("="), RB, LB, EQ, RB]
+tb "[=][=][=]"      => [Func("=", Some("="), true), Func("=", None, true)]
-t "[=][[=][=][=]]" => [LB, EQ, RB, LB, LB, EQ, RB, LB, T("="), RB, LB, EQ, RB, RB]
+tb "[=][[=][=][=]]" => [Func("=", Some("[=][=][=]")), true]
 
 // Escapes.
-t r"\["   => [T("[")]
+tb r"\[" => [T("[")]
-t r"\]"   => [T("]")]
+tb r"\]" => [T("]")]
-t r"\\"   => [T(r"\")]
+tb r"\\" => [T(r"\")]
-t r"\/"   => [T("/")]
+tb r"\/" => [T("/")]
-t r"\*"   => [T("*")]
+tb r"\*" => [T("*")]
-t r"\_"   => [T("_")]
+tb r"\_" => [T("_")]
-t r"\`"   => [T("`")]
+tb r"\`" => [T("`")]
 
 // Unescapable special symbols.
-t r"\:"   => [T(r"\"), T(":")]
+th r"\:" => [T(r"\"), T(":")]
-t r"\="   => [T(r"\"), T("=")]
+th r"\=" => [T(r"\"), T("=")]
-t r"[\:]" => [LB, T(r"\"), CL, RB]
+th r"\:" => [T(r"\"), Colon]
-t r"[\=]" => [LB, T(r"\"), EQ, RB]
+th r"\=" => [T(r"\"), Equals]
-t r"[\,]" => [LB, T(r"\"), CM, RB]
+th r"\," => [T(r"\"), Comma]
 
-// Spans
+// Spans.
-ts "hello"           => [(0:0, 0:5, T("hello"))]
+tbs "hello"          => [(0:0, 0:5, T("hello"))]
-ts "ab\r\nc"         => [(0:0, 0:2, T("ab")), (0:2, 1:0, W(1)), (1:0, 1:1, T("c"))]
+tbs "ab\r\nc"        => [(0:0, 0:2, T("ab")), (0:2, 1:0, S(1)), (1:0, 1:1, T("c"))]
-ts "[a=10]"          => [(0:0, 0:1, LB), (0:1, 0:2, ID("a")), (0:2, 0:3, EQ),
+tbs "[x = \"(1)\"]*" => [(0:0, 0:11, Func("x = \"(1)\"", None, true)), (0:11, 0:12, Star)]
-                         (0:3, 0:5, NUM(10.0)), (0:5, 0:6, RB)]
+tbs "// ab\r\n\nf"   => [(0:0, 0:5, LC(" ab")), (0:5, 2:0, S(2)), (2:0, 2:1, T("f"))]
-ts r#"[x = "(1)"]*"# => [(0:0, 0:1, LB), (0:1, 0:2, ID("x")), (0:2, 0:3, W(0)),
+tbs "/*b*/_"         => [(0:0, 0:5, BC("b")), (0:5, 0:6, Underscore)]
-                         (0:3, 0:4, EQ), (0:4, 0:5, W(0)), (0:5, 0:10, STR("(1)")),
+ths "a=10"           => [(0:0, 0:1, Id("a")), (0:1, 0:2, Equals), (0:2, 0:4, Num(10.0))]
-                         (0:10, 0:11, RB), (0:11, 0:12, S)]
-ts "// ab\r\n\nf"    => [(0:0, 0:5, LC(" ab")), (0:5, 2:0, W(2)), (2:0, 2:1, T("f"))]
-ts "/*b*/_"          => [(0:0, 0:5, BC("b")), (0:5, 0:6, U)]

tests/src/layouter.rs

@@ -10,8 +10,8 @@ use futures_executor::block_on;
 
 use typstc::Typesetter;
 use typstc::layout::{MultiLayout, Serialize};
-use typstc::size::{Size, Size2D};
+use typstc::size::{Size, Size2D, ValueBox};
-use typstc::style::PageStyle;
+use typstc::style::{PageStyle, PaperClass};
 use typstc::toddle::query::FileSystemFontProvider;
 use typstc::export::pdf::PdfExporter;
 
@@ -62,14 +62,15 @@ fn main() -> DynResult<()> {
     Ok(())
 }
 
-/// Create a _PDF_ with a name from the source code.
+/// Create a _PDF_ and render with a name from the source code.
 fn test(name: &str, src: &str) -> DynResult<()> {
     println!("Testing: {}.", name);
 
     let mut typesetter = Typesetter::new();
     typesetter.set_page_style(PageStyle {
+        class: PaperClass::Custom,
         dimensions: Size2D::with_all(Size::pt(250.0)),
-        .. PageStyle::default()
+        margins: ValueBox::with_all(None),
     });
 
     let provider = FileSystemFontProvider::from_index("../fonts/index.json")?;
@@ -120,31 +121,43 @@ fn test(name: &str, src: &str) -> DynResult<()> {
 
 /// Compile the source code with the typesetter.
 fn compile(typesetter: &Typesetter, src: &str) -> MultiLayout {
-    #[cfg(not(debug_assertions))] {
+    #![allow(unused_variables)]
-        use std::time::Instant;
+    use std::time::Instant;
 
-        // Warmup.
+    // Warmup.
+    #[cfg(not(debug_assertions))]
+    let warmup = {
         let warmup_start = Instant::now();
         block_on(typesetter.typeset(&src));
-        let warmup_end = Instant::now();
+        Instant::now() - warmup_start
+    };
 
-        let start = Instant::now();
+    let start = Instant::now();
-        let tree = typesetter.parse(&src).output;
+    let parsed = typesetter.parse(&src);
-        let mid = Instant::now();
+    let parse = Instant::now() - start;
-        let layouts = block_on(typesetter.layout(&tree)).output;
-        let end = Instant::now();
 
-        println!(" - cold start:  {:?}", warmup_end - warmup_start);
+    if !parsed.errors.is_empty() {
-        println!(" - warmed up:   {:?}", end - start);
+        println!("parse errors: {:#?}", parsed.errors);
-        println!("   - parsing:   {:?}", mid - start);
-        println!("   - layouting: {:?}", end - mid);
-        println!();
-
-        layouts
     }
 
-    #[cfg(debug_assertions)]
+    let start_layout = Instant::now();
-    block_on(typesetter.typeset(&src))
+    let layouted = block_on(typesetter.layout(&parsed.output));
+    let layout = Instant::now() - start_layout;
+    let total = Instant::now() - start;
+
+    if !layouted.errors.is_empty() {
+        println!("layout errors: {:#?}", layouted.errors);
+    }
+
+    #[cfg(not(debug_assertions))] {
+        println!(" - cold start:  {:?}", warmup);
+        println!(" - warmed up:   {:?}", total);
+        println!("   - parsing:   {:?}", parse);
+        println!("   - layouting: {:?}", layout);
+        println!();
+    }
+
+    layouted.output
 }
 
 /// Command line options.