Merge pull request #7 from typst/parser-update

Parser update

src/func.rs

@@ -1,9 +1,8 @@
 //! Trait and prelude for custom functions.
 
 use crate::Pass;
-use crate::syntax::ParseContext;
+use crate::syntax::ParseState;
-use crate::syntax::func::FuncHeader;
+use crate::syntax::func::FuncCall;
-use crate::syntax::span::Spanned;
 
 /// Types that are useful for creating your own functions.
 pub mod prelude {
@@ -17,7 +16,6 @@ pub mod prelude {
     pub use crate::syntax::span::{Span, Spanned};
 }
 
-
 /// Parse a function from source code.
 pub trait ParseFunc {
     /// A metadata type whose value is passed into the function parser. This
@@ -33,9 +31,8 @@ pub trait ParseFunc {
 
     /// Parse the header and body into this function given a context.
     fn parse(
-        header: FuncHeader,
+        header: FuncCall,
-        body: Option<Spanned<&str>>,
+        state: &ParseState,
-        ctx: ParseContext,
         metadata: Self::Meta,
     ) -> Pass<Self> where Self: Sized;
 }
@@ -53,8 +50,8 @@ pub trait ParseFunc {
 ///         body: Option<SyntaxModel>,
 ///     }
 ///
-///     parse(header, body, ctx, f) {
+///     parse(header, body, state, f) {
-///         let body = body!(opt: body, ctx, f);
+///         let body = body!(opt: body, state, f);
 ///         let hidden = header.args.pos.get::<bool>(&mut f.problems)
 ///             .or_missing(&mut f.problems, header.name.span, "hidden")
 ///             .unwrap_or(false);
@@ -112,7 +109,7 @@ macro_rules! function {
     (@parse($name:ident, $meta:ty) parse(
         $header:ident,
         $body:ident,
-        $ctx:ident,
+        $state:ident,
         $feedback:ident,
         $metadata:ident
     ) $code:block $($r:tt)*) => {
@@ -120,18 +117,18 @@ macro_rules! function {
             type Meta = $meta;
 
             fn parse(
-                #[allow(unused)] mut header: $crate::syntax::func::FuncHeader,
+                #[allow(unused)] mut call: $crate::syntax::func::FuncCall,
-                #[allow(unused)] $body: Option<$crate::syntax::span::Spanned<&str>>,
+                #[allow(unused)] $state: &$crate::syntax::ParseState,
-                #[allow(unused)] $ctx: $crate::syntax::ParseContext,
                 #[allow(unused)] $metadata: Self::Meta,
             ) -> $crate::Pass<Self> where Self: Sized {
                 let mut feedback = $crate::Feedback::new();
-                #[allow(unused)] let $header = &mut header;
+                #[allow(unused)] let $header = &mut call.header;
+                #[allow(unused)] let $body = &mut call.body;
                 #[allow(unused)] let $feedback = &mut feedback;
 
                 let func = $code;
 
-                for arg in header.args.into_iter() {
+                for arg in call.header.args.into_iter() {
                     error!(@feedback, arg.span, "unexpected argument");
                 }
 
@@ -167,21 +164,20 @@ macro_rules! function {
 /// Parse the body of a function.
 ///
 /// - If the function does not expect a body, use `body!(nope: body, feedback)`.
-/// - If the function can have a body, use `body!(opt: body, ctx, feedback,
+/// - If the function can have a body, use `body!(opt: body, state, feedback,
 ///   decos)`.
 ///
 /// # Arguments
 /// - The `$body` should be of type `Option<Spanned<&str>>`.
-/// - The `$ctx` is the [`ParseContext`](crate::syntax::ParseContext) to use for
+/// - The `$state` is the parse state to use.
-///   parsing.
 /// - The `$feedback` should be a mutable references to a
 ///   [`Feedback`](crate::Feedback) struct which is filled with the feedback
 ///   information arising from parsing.
 #[macro_export]
 macro_rules! body {
-    (opt: $body:expr, $ctx:expr, $feedback:expr) => ({
+    (opt: $body:expr, $state:expr, $feedback:expr) => ({
         $body.map(|body| {
-            let parsed = $crate::syntax::parse(body.span.start, body.v, $ctx);
+            let parsed = $crate::syntax::parse(body.v, body.span.start, $state);
             $feedback.extend(parsed.feedback);
             parsed.output
         })

src/layout/model.rs

@@ -142,7 +142,7 @@ impl<'a> ModelLayouter<'a> {
         }).await;
 
         // Add the errors generated by the model to the error list.
-        self.feedback.extend_offset(model.span.start, layouted.feedback);
+        self.feedback.extend_offset(layouted.feedback, model.span.start);
 
         for command in layouted.output {
             self.execute_command(command, model.span).await;

src/lib.rs

@@ -30,7 +30,7 @@ use toddle::query::{FontProvider, FontIndex, FontDescriptor};
 use crate::problem::Problems;
 use crate::layout::MultiLayout;
 use crate::style::{LayoutStyle, PageStyle, TextStyle};
-use crate::syntax::{SyntaxModel, Scope, Decoration, ParseContext, parse};
+use crate::syntax::{SyntaxModel, Scope, Decoration, ParseState, parse};
 use crate::syntax::span::{Position, SpanVec, offset_spans};
 
 
@@ -42,6 +42,8 @@ macro_rules! pub_use_mod {
     };
 }
 
+#[macro_use]
+mod macros;
 #[macro_use]
 pub mod problem;
 pub mod export;
@@ -57,14 +59,13 @@ pub mod syntax;
 /// Transforms source code into typesetted layouts.
 ///
 /// A typesetter can be configured through various methods.
-#[derive(Debug)]
 pub struct Typesetter {
     /// The font loader shared by all typesetting processes.
     loader: GlobalFontLoader,
     /// The base layouting style.
     style: LayoutStyle,
-    /// The standard library scope.
+    /// The base parser state.
-    scope: Scope,
+    parse_state: ParseState,
     /// Whether to render debug boxes.
     debug: bool,
 }
@@ -84,7 +85,7 @@ impl Typesetter {
         Typesetter {
             loader: RefCell::new(FontLoader::new(provider)),
             style: LayoutStyle::default(),
-            scope: Scope::with_std(),
+            parse_state: ParseState { scope: Scope::with_std() },
             debug: false,
         }
     }
@@ -111,7 +112,7 @@ impl Typesetter {
 
     /// Parse source code into a syntax tree.
     pub fn parse(&self, src: &str) -> Pass<SyntaxModel> {
-        parse(Position::ZERO, src, ParseContext { scope: &self.scope })
+        parse(src, Position::ZERO, &self.parse_state)
     }
 
     /// Layout a syntax tree and return the produced layout.
@@ -203,10 +204,10 @@ impl Feedback {
     }
 
     /// Add more feedback whose spans are local and need to be offset by an
-    /// `offset` to be correct for this feedbacks context.
+    /// `offset` to be correct in this feedback's context.
-    pub fn extend_offset(&mut self, offset: Position, other: Feedback) {
+    pub fn extend_offset(&mut self, more: Feedback, offset: Position) {
-        self.problems.extend(offset_spans(offset, other.problems));
+        self.problems.extend(offset_spans(more.problems, offset));
-        self.decos.extend(offset_spans(offset, other.decos));
+        self.decos.extend(offset_spans(more.decos, offset));
     }
 }
 

src/library/mod.rs

@@ -53,10 +53,10 @@ function! {
         body: Option<SyntaxModel>,
     }
 
-    parse(header, body, ctx, f) {
+    parse(header, body, state, f) {
         header.args.pos.items.clear();
         header.args.key.pairs.clear();
-        ValFunc { body: body!(opt: body, ctx, f) }
+        ValFunc { body: body!(opt: body, state, f) }
     }
 
     layout(self, ctx, f) {

src/library/page.rs

@@ -12,7 +12,7 @@ function! {
         flip: bool,
     }
 
-    parse(header, body, ctx, f) {
+    parse(header, body, state, f) {
         body!(nope: body, f);
         PageSizeFunc {
             paper: header.args.pos.get::<Paper>(&mut f.problems),
@@ -50,7 +50,7 @@ function! {
         padding: PaddingMap,
     }
 
-    parse(header, body, ctx, f) {
+    parse(header, body, state, f) {
         body!(nope: body, f);
         PageMarginsFunc {
             padding: PaddingMap::parse(&mut f.problems, &mut header.args),

src/library/spacing.rs

@@ -46,9 +46,9 @@ function! {
 
     type Meta = ContentKind;
 
-    parse(header, body, ctx, f, meta) {
+    parse(header, body, state, f, meta) {
         ContentSpacingFunc {
-            body: body!(opt: body, ctx, f),
+            body: body!(opt: body, state, f),
             content: meta,
             spacing: header.args.pos.get::<f64>(&mut f.problems)
                 .map(|num| num as f32)
@@ -84,7 +84,7 @@ function! {
 
     type Meta = Option<SpecificAxis>;
 
-    parse(header, body, ctx, f, meta) {
+    parse(header, body, state, f, meta) {
         body!(nope: body, f);
         SpacingFunc {
             spacing: if let Some(axis) = meta {

src/macros.rs

@@ -0,0 +1,25 @@
+#![allow(unused)]
+
+/// Unwrap the result if it is `Ok(T)` or evaluate `$or` if it is `Err(_)`.
+/// This fits use cases the `?`-operator does not cover, like:
+/// ```
+/// try_or!(result, continue);
+/// ```
+macro_rules! try_or {
+    ($result:expr, $or:expr $(,)?) => {
+        match $result {
+            Ok(v) => v,
+            Err(_) => { $or }
+        }
+    };
+}
+
+/// Unwrap the option if it is `Some(T)` or evaluate `$or` if it is `None`.
+macro_rules! try_opt_or {
+    ($option:expr, $or:expr $(,)?) => {
+        match $option {
+            Some(v) => v,
+            None => { $or }
+        }
+    };
+}

src/syntax/expr.rs

@@ -93,9 +93,6 @@ impl Debug for Expr {
 
 /// A unicode identifier.
 ///
-/// The identifier must be valid! This is checked in [`Ident::new`] or
-/// [`is_identifier`].
-///
 /// # Example
 /// ```typst
 /// [func: "hi", ident]
@@ -105,7 +102,8 @@ impl Debug for Expr {
 pub struct Ident(pub String);
 
 impl Ident {
-    /// Create a new identifier from a string checking that it is valid.
+    /// Create a new identifier from a string checking that it is a valid
+    /// unicode identifier.
     pub fn new<S>(ident: S) -> Option<Ident> where S: AsRef<str> + Into<String> {
         if is_identifier(ident.as_ref()) {
             Some(Ident(ident.into()))

src/syntax/func/mod.rs

@@ -9,22 +9,23 @@ pub_use_mod!(maps);
 pub_use_mod!(keys);
 pub_use_mod!(values);
 
+/// An invocation of a function.
+#[derive(Debug, Clone, PartialEq)]
+pub struct FuncCall<'s> {
+    pub header: FuncHeader,
+    /// The body as a raw string containing what's inside of the brackets.
+    pub body: Option<Spanned<&'s str>>,
+}
 
-/// The parsed header of a function.
+/// The parsed header of a function (everything in the first set of brackets).
 #[derive(Debug, Clone, PartialEq)]
 pub struct FuncHeader {
-    /// The function name, that is:
-    /// ```typst
-    /// [box: w=5cm]
-    ///  ^^^
-    /// ```
     pub name: Spanned<Ident>,
-    /// The arguments passed to the function.
     pub args: FuncArgs,
 }
 
 /// The positional and keyword arguments passed to a function.
-#[derive(Debug, Clone, PartialEq)]
+#[derive(Debug, Default, Clone, PartialEq)]
 pub struct FuncArgs {
     /// The positional arguments.
     pub pos: Tuple,

src/syntax/scope.rs

@@ -5,12 +5,10 @@ use std::fmt::{self, Debug, Formatter};
 
 use crate::Pass;
 use crate::func::ParseFunc;
-use super::func::FuncHeader;
+use super::func::FuncCall;
-use super::parsing::ParseContext;
+use super::parsing::ParseState;
-use super::span::Spanned;
 use super::Model;
 
-
 /// A map from identifiers to function parsers.
 pub struct Scope {
     parsers: HashMap<String, Box<Parser>>,
@@ -50,10 +48,8 @@ impl Scope {
     }
 
     /// Return the parser with the given name if there is one.
-    pub fn get_parser(&self, name: &str) -> Result<&Parser, &Parser> {
+    pub fn get_parser(&self, name: &str) -> Option<&Parser> {
-        self.parsers.get(name)
+        self.parsers.get(name).map(AsRef::as_ref)
-            .map(|x| &**x)
-            .ok_or_else(|| &*self.fallback)
     }
 
     /// Return the fallback parser.
@@ -72,16 +68,12 @@ impl Debug for Scope {
 
 /// A function which parses the source of a function into a model type which
 /// implements [`Model`].
-type Parser = dyn Fn(
+type Parser = dyn Fn(FuncCall, &ParseState) -> Pass<Box<dyn Model>>;
-    FuncHeader,
-    Option<Spanned<&str>>,
-    ParseContext,
-) -> Pass<Box<dyn Model>>;
 
 fn parser<F>(metadata: <F as ParseFunc>::Meta) -> Box<Parser>
 where F: ParseFunc + Model + 'static {
-    Box::new(move |h, b, c| {
+    Box::new(move |f, s| {
-        F::parse(h, b, c, metadata.clone())
+        F::parse(f, s, metadata.clone())
             .map(|model| Box::new(model) as Box<dyn Model>)
     })
 }

src/syntax/span.rs

@@ -4,6 +4,119 @@ use std::fmt::{self, Debug, Formatter};
 use std::ops::{Add, Sub};
 use serde::Serialize;
 
+/// A vector of spanned values of type `T`.
+pub type SpanVec<T> = Vec<Spanned<T>>;
+
+/// [Offset](Span::offset) all spans in a vector of spanned things by a start
+/// position.
+pub fn offset_spans<T>(
+    vec: SpanVec<T>,
+    start: Position,
+) -> impl Iterator<Item=Spanned<T>> {
+    vec.into_iter().map(move |s| s.map_span(|span| span.offset(start)))
+}
+
+/// A value with the span it corresponds to in the source code.
+#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)]
+pub struct Spanned<T> {
+    /// The value.
+    pub v: T,
+    /// The corresponding span.
+    pub span: Span,
+}
+
+impl<T> Spanned<T> {
+    /// Create a new instance from a value and its span.
+    pub fn new(v: T, span: Span) -> Spanned<T> {
+        Spanned { v, span }
+    }
+
+    /// Create a new instance from a value with the zero span.
+    pub fn zero(v: T) -> Spanned<T> {
+        Spanned { v, span: Span::ZERO }
+    }
+
+    /// Access the value.
+    pub fn value(self) -> T {
+        self.v
+    }
+
+    /// Map the value using a function while keeping the span.
+    pub fn map<V, F>(self, f: F) -> Spanned<V> where F: FnOnce(T) -> V {
+        Spanned { v: f(self.v), span: self.span }
+    }
+
+    /// Maps the span while keeping the value.
+    pub fn map_span<F>(mut self, f: F) -> Spanned<T> where F: FnOnce(Span) -> Span {
+        self.span = f(self.span);
+        self
+    }
+}
+
+impl<T: Debug> Debug for Spanned<T> {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        self.v.fmt(f)?;
+        if f.alternate() {
+            f.write_str(" ")?;
+            self.span.fmt(f)?;
+        }
+        Ok(())
+    }
+}
+
+/// Locates a slice of source code.
+#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)]
+pub struct Span {
+    /// The inclusive start position.
+    pub start: Position,
+    /// The inclusive end position.
+    pub end: Position,
+}
+
+impl Span {
+    /// The zero span.
+    pub const ZERO: Span = Span { start: Position::ZERO, end: Position::ZERO };
+
+    /// Create a new span from start and end positions.
+    pub fn new(start: Position, end: Position) -> Span {
+        Span { start, end }
+    }
+
+    /// Create a new span with the earlier start and later end position.
+    pub fn merge(a: Span, b: Span) -> Span {
+        Span {
+            start: a.start.min(b.start),
+            end: a.end.max(b.end),
+        }
+    }
+
+    /// Create a span including just a single position.
+    pub fn at(pos: Position) -> Span {
+        Span { start: pos, end: pos }
+    }
+
+    /// Expand a span by merging it with another span.
+    pub fn expand(&mut self, other: Span) {
+        *self = Span::merge(*self, other)
+    }
+
+    /// Offset a span by a start position.
+    ///
+    /// This is, for example, used to translate error spans from function local
+    /// to global.
+    pub fn offset(self, start: Position) -> Span {
+        Span {
+            start: start + self.start,
+            end: start + self.end,
+        }
+    }
+}
+
+impl Debug for Span {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        write!(f, "<{:?}-{:?}>", self.start, self.end)
+    }
+}
 
 /// Zero-indexed line-column position in source code.
 #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash, Serialize)]
@@ -18,7 +131,7 @@ impl Position {
     /// The line 0, column 0 position.
     pub const ZERO: Position = Position { line: 0, column: 0 };
 
-    /// Crete a new instance from line and column.
+    /// Create a new position from line and column.
     pub fn new(line: usize, column: usize) -> Position {
         Position { line, column }
     }
@@ -65,102 +178,3 @@ impl Debug for Position {
         write!(f, "{}:{}", self.line, self.column)
     }
 }
-
-/// Locates a slice of source code.
-#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)]
-pub struct Span {
-    /// The inclusive start position.
-    pub start: Position,
-    /// The inclusive end position.
-    pub end: Position,
-}
-
-impl Span {
-    /// The zero span.
-    pub const ZERO: Span = Span { start: Position::ZERO, end: Position::ZERO };
-
-    /// Create a new span from start and end positions.
-    pub fn new(start: Position, end: Position) -> Span {
-        Span { start, end }
-    }
-
-    /// Create a new span with the earlier start and later end position.
-    pub fn merge(a: Span, b: Span) -> Span {
-        Span {
-            start: a.start.min(b.start),
-            end: a.end.max(b.end),
-        }
-    }
-
-    /// Create a span including just a single position.
-    pub fn at(pos: Position) -> Span {
-        Span { start: pos, end: pos }
-    }
-
-    /// Offset a span by a start position.
-    ///
-    /// This is, for example, used to translate error spans from function local
-    /// to global.
-    pub fn offset(self, start: Position) -> Span {
-        Span {
-            start: start + self.start,
-            end: start + self.end,
-        }
-    }
-}
-
-impl Debug for Span {
-    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
-        write!(f, "({:?} -> {:?})", self.start, self.end)
-    }
-}
-
-/// A value with the span it corresponds to in the source code.
-#[derive(Copy, Clone, Eq, PartialEq, Hash, Serialize)]
-pub struct Spanned<T> {
-    /// The value.
-    pub v: T,
-    /// The corresponding span.
-    pub span: Span,
-}
-
-impl<T> Spanned<T> {
-    /// Create a new instance from a value and its span.
-    pub fn new(v: T, span: Span) -> Spanned<T> {
-        Spanned { v, span }
-    }
-
-    /// Access the value.
-    pub fn value(self) -> T {
-        self.v
-    }
-
-    /// Map the value using a function while keeping the span.
-    pub fn map<V, F>(self, f: F) -> Spanned<V> where F: FnOnce(T) -> V {
-        Spanned { v: f(self.v), span: self.span }
-    }
-
-    /// Maps the span while keeping the value.
-    pub fn map_span<F>(mut self, f: F) -> Spanned<T> where F: FnOnce(Span) -> Span {
-        self.span = f(self.span);
-        self
-    }
-}
-
-impl<T: Debug> Debug for Spanned<T> {
-    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
-        self.v.fmt(f)?;
-        f.write_str(" ")?;
-        self.span.fmt(f)
-    }
-}
-
-/// A vector of spanned things.
-pub type SpanVec<T> = Vec<Spanned<T>>;
-
-/// [Offset](Span::offset) all spans in a vector of spanned things by a start
-/// position.
-pub fn offset_spans<T>(start: Position, vec: SpanVec<T>) -> impl Iterator<Item=Spanned<T>> {
-    vec.into_iter()
-        .map(move |s| s.map_span(|span| span.offset(start)))
-}

src/syntax/test.rs

@@ -1,17 +1,14 @@
 use std::fmt::Debug;
 
 use super::func::FuncHeader;
+use super::span::Spanned;
 use super::expr::{Expr, Tuple, NamedTuple, Object};
-use super::span::{Span, Spanned};
-use super::tokens::Token;
 use super::*;
 
-
+/// Check whether the expected and found results are the same.
-/// Check whether the expected and found results for the given source code
+pub fn check<T>(src: &str, exp: T, found: T, cmp_spans: bool)
-/// match by the comparison function, and print them out otherwise.
-pub fn check<T>(src: &str, exp: T, found: T, spans: bool)
 where T: Debug + PartialEq + SpanlessEq {
-    let cmp = if spans { PartialEq::eq } else { SpanlessEq::spanless_eq };
+    let cmp = if cmp_spans { PartialEq::eq } else { SpanlessEq::spanless_eq };
     if !cmp(&exp, &found) {
         println!("source:   {:?}", src);
         println!("expected: {:#?}", exp);
@@ -23,16 +20,25 @@ where T: Debug + PartialEq + SpanlessEq {
 /// Create a vector of optionally spanned expressions from a list description.
 ///
 /// # Examples
-/// When you want to add span information to the items, the format is as
-/// follows.
 /// ```
+/// // With spans
 /// spanned![(0:0, 0:5, "hello"), (0:5, 0:3, "world")]
+///
+/// // Without spans: Implicit zero spans.
+/// spanned!["hello", "world"]
 /// ```
-/// The span information can simply be omitted to create a vector with items
+macro_rules! span_vec {
-/// that are spanned with zero spans.
+    ($(($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)),* $(,)?) => {
-macro_rules! spanned {
+        (vec![$(span_item!(($sl:$sc, $el:$ec, $v))),*], true)
-    (item ($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)) => ({
+    };
-        #[allow(unused_imports)]
+
+    ($($v:expr),* $(,)?) => {
+        (vec![$(span_item!($v)),*], false)
+    };
+}
+
+macro_rules! span_item {
+    (($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)) => ({
         use $crate::syntax::span::{Position, Span, Spanned};
         Spanned {
             span: Span::new(
@@ -43,22 +49,9 @@ macro_rules! spanned {
         }
     });
 
-    (item $v:expr) => {
+    ($v:expr) => {
-        $crate::syntax::test::zspan($v)
+        $crate::syntax::span::Spanned::zero($v)
     };
-
-    (vec $(($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)),* $(,)?) => {
-        (vec![$(spanned![item ($sl:$sc, $el:$ec, $v)]),*], true)
-    };
-
-    (vec $($v:expr),* $(,)?) => {
-        (vec![$($crate::syntax::test::zspan($v)),*], false)
-    };
-}
-
-/// Span an element with a zero span.
-pub fn zspan<T>(v: T) -> Spanned<T> {
-    Spanned { v, span: Span::ZERO }
 }
 
 function! {
@@ -70,13 +63,13 @@ function! {
         pub body: Option<SyntaxModel>,
     }
 
-    parse(header, body, ctx, f) {
+    parse(header, body, state, f) {
         let cloned = header.clone();
         header.args.pos.items.clear();
         header.args.key.pairs.clear();
         DebugFn {
             header: cloned,
-            body: body!(opt: body, ctx, f),
+            body: body!(opt: body, state, f),
         }
     }
 
@@ -120,8 +113,8 @@ impl SpanlessEq for DebugFn {
 impl SpanlessEq for Expr {
     fn spanless_eq(&self, other: &Expr) -> bool {
         match (self, other) {
-            (Expr::NamedTuple(a), Expr::NamedTuple(b)) => a.spanless_eq(b),
             (Expr::Tuple(a), Expr::Tuple(b)) => a.spanless_eq(b),
+            (Expr::NamedTuple(a), Expr::NamedTuple(b)) => a.spanless_eq(b),
             (Expr::Object(a), Expr::Object(b)) => a.spanless_eq(b),
             (Expr::Neg(a), Expr::Neg(b)) => a.spanless_eq(&b),
             (Expr::Add(a1, a2), Expr::Add(b1, b2)) => a1.spanless_eq(&b1) && a2.spanless_eq(&b2),
@@ -175,8 +168,7 @@ impl<T: SpanlessEq> SpanlessEq for Box<T> {
     }
 }
 
-/// Implement `SpanlessEq` by just forwarding to `PartialEq`.
+macro_rules! impl_through_partial_eq {
-macro_rules! forward {
     ($type:ty) => {
         impl SpanlessEq for $type {
             fn spanless_eq(&self, other: &$type) -> bool {
@@ -186,6 +178,8 @@ macro_rules! forward {
     };
 }
 
-forward!(String);
+impl_through_partial_eq!(Token<'_>);
-forward!(Token<'_>);
+
-forward!(Decoration);
+// Implement for string and decoration to be able to compare feedback.
+impl_through_partial_eq!(String);
+impl_through_partial_eq!(Decoration);

src/syntax/tokens.rs

@@ -5,9 +5,8 @@ use unicode_xid::UnicodeXID;
 use crate::size::Size;
 use super::span::{Position, Span, Spanned};
 
-use self::Token::*;
+use Token::*;
-use self::TokenizationMode::*;
+use TokenMode::*;
-
 
 /// A minimal semantic entity of source code.
 #[derive(Debug, Copy, Clone, PartialEq)]
@@ -152,7 +151,7 @@ impl<'s> Token<'s> {
 #[derive(Debug)]
 pub struct Tokens<'s> {
     src: &'s str,
-    mode: TokenizationMode,
+    mode: TokenMode,
     iter: Peekable<Chars<'s>>,
     position: Position,
     index: usize,
@@ -163,20 +162,22 @@ pub struct Tokens<'s> {
 /// backtick tokens.
 #[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
 #[allow(missing_docs)]
-pub enum TokenizationMode {
+pub enum TokenMode {
     Header,
     Body,
 }
 
 impl<'s> Tokens<'s> {
-    /// Create a new token iterator with the given mode where the first token
+    /// Create a new token iterator with the given mode.
-    /// span starts an the given `start` position.
+    ///
-    pub fn new(start: Position, src: &'s str, mode: TokenizationMode) -> Tokens<'s> {
+    /// The first token's span starts an the given `offset` position instead of
+    /// the zero position.
+    pub fn new(src: &'s str, offset: Position, mode: TokenMode) -> Tokens<'s> {
         Tokens {
             src,
             mode,
             iter: src.chars().peekable(),
-            position: start,
+            position: offset,
             index: 0,
         }
     }
@@ -188,7 +189,7 @@ impl<'s> Tokens<'s> {
     }
 
     /// The line-colunn position in the source at which the last token ends and
-    /// next token will start. This position is
+    /// next token will start.
     pub fn pos(&self) -> Position {
         self.position
     }
@@ -204,15 +205,15 @@ impl<'s> Iterator for Tokens<'s> {
 
         let token = match first {
             // Comments.
-            '/' if self.peek() == Some('/') => self.parse_line_comment(),
+            '/' if self.peek() == Some('/') => self.read_line_comment(),
-            '/' if self.peek() == Some('*') => self.parse_block_comment(),
+            '/' if self.peek() == Some('*') => self.read_block_comment(),
             '*' if self.peek() == Some('/') => { self.eat(); Invalid("*/") }
 
             // Whitespace.
-            c if c.is_whitespace() => self.parse_whitespace(start),
+            c if c.is_whitespace() => self.read_whitespace(start),
 
             // Functions.
-            '[' => self.parse_function(start),
+            '[' => self.read_function(start),
             ']' => Invalid("]"),
 
             // Syntactic elements in function headers.
@@ -230,7 +231,7 @@ impl<'s> Iterator for Tokens<'s> {
             '/' if self.mode == Header => Slash,
 
             // String values.
-            '"' if self.mode == Header => self.parse_string(),
+            '"' if self.mode == Header => self.read_string(),
 
             // Star serves a double purpose as a style modifier
             // and a expression operator in the header.
@@ -238,13 +239,13 @@ impl<'s> Iterator for Tokens<'s> {
 
             // Style toggles.
             '_' if self.mode == Body => Underscore,
-            '`' if self.mode == Body => self.parse_raw(),
+            '`' if self.mode == Body => self.read_raw(),
 
             // An escaped thing.
-            '\\' if self.mode == Body => self.parse_escaped(),
+            '\\' if self.mode == Body => self.read_escaped(),
 
             // A hex expression.
-            '#' if self.mode == Header => self.parse_hex_value(),
+            '#' if self.mode == Header => self.read_hex(),
 
             // Expressions or just strings.
             c => {
@@ -267,7 +268,7 @@ impl<'s> Iterator for Tokens<'s> {
                 }, false, -(c.len_utf8() as isize), 0).0;
 
                 if self.mode == Header {
-                    self.parse_expr(text)
+                    self.read_expr(text)
                 } else {
                     Text(text)
                 }
@@ -282,11 +283,11 @@ impl<'s> Iterator for Tokens<'s> {
 }
 
 impl<'s> Tokens<'s> {
-    fn parse_line_comment(&mut self) -> Token<'s> {
+    fn read_line_comment(&mut self) -> Token<'s> {
         LineComment(self.read_string_until(is_newline_char, false, 1, 0).0)
     }
 
-    fn parse_block_comment(&mut self) -> Token<'s> {
+    fn read_block_comment(&mut self) -> Token<'s> {
         enum Last { Slash, Star, Other }
 
         self.eat();
@@ -314,14 +315,14 @@ impl<'s> Tokens<'s> {
         }, true, 0, -2).0)
     }
 
-    fn parse_whitespace(&mut self, start: Position) -> Token<'s> {
+    fn read_whitespace(&mut self, start: Position) -> Token<'s> {
         self.read_string_until(|n| !n.is_whitespace(), false, 0, 0);
         let end = self.pos();
 
         Space(end.line - start.line)
     }
 
-    fn parse_function(&mut self, start: Position) -> Token<'s> {
+    fn read_function(&mut self, start: Position) -> Token<'s> {
         let (header, terminated) = self.read_function_part(Header);
         self.eat();
 
@@ -341,7 +342,7 @@ impl<'s> Tokens<'s> {
         Function { header, body: Some(Spanned { v: body, span }), terminated }
     }
 
-    fn read_function_part(&mut self, mode: TokenizationMode) -> (&'s str, bool) {
+    fn read_function_part(&mut self, mode: TokenMode) -> (&'s str, bool) {
         let start = self.index();
         let mut terminated = false;
 
@@ -353,11 +354,11 @@ impl<'s> Tokens<'s> {
 
             self.eat();
             match n {
-                '[' => { self.parse_function(Position::ZERO); }
+                '[' => { self.read_function(Position::ZERO); }
-                '/' if self.peek() == Some('/') => { self.parse_line_comment(); }
+                '/' if self.peek() == Some('/') => { self.read_line_comment(); }
-                '/' if self.peek() == Some('*') => { self.parse_block_comment(); }
+                '/' if self.peek() == Some('*') => { self.read_block_comment(); }
-                '"' if mode == Header => { self.parse_string(); }
+                '"' if mode == Header => { self.read_string(); }
-                '`' if mode == Body => { self.parse_raw(); }
+                '`' if mode == Body => { self.read_raw(); }
                 '\\' => { self.eat(); }
                 _ => {}
             }
@@ -367,12 +368,12 @@ impl<'s> Tokens<'s> {
         (&self.src[start .. end], terminated)
     }
 
-    fn parse_string(&mut self) -> Token<'s> {
+    fn read_string(&mut self) -> Token<'s> {
         let (string, terminated) = self.read_until_unescaped('"');
         ExprStr { string, terminated }
     }
 
-    fn parse_raw(&mut self) -> Token<'s> {
+    fn read_raw(&mut self) -> Token<'s> {
         let (raw, terminated) = self.read_until_unescaped('`');
         Raw { raw, terminated }
     }
@@ -390,7 +391,7 @@ impl<'s> Tokens<'s> {
         }, true, 0, -1)
     }
 
-    fn parse_escaped(&mut self) -> Token<'s> {
+    fn read_escaped(&mut self) -> Token<'s> {
         fn is_escapable(c: char) -> bool {
             match c {
                 '[' | ']' | '\\' | '/' | '*' | '_' | '`' | '"' => true,
@@ -410,7 +411,7 @@ impl<'s> Tokens<'s> {
         }
     }
 
-    fn parse_hex_value(&mut self) -> Token<'s> {
+    fn read_hex(&mut self) -> Token<'s> {
         // This will parse more than the permissable 0-9, a-f, A-F character
         // ranges to provide nicer error messages later.
         ExprHex(self.read_string_until(
@@ -419,7 +420,7 @@ impl<'s> Tokens<'s> {
         ).0)
     }
 
-    fn parse_expr(&mut self, text: &'s str) -> Token<'s> {
+    fn read_expr(&mut self, text: &'s str) -> Token<'s> {
         if let Ok(b) = text.parse::<bool>() {
             ExprBool(b)
         } else if let Ok(num) = text.parse::<f64>() {
@@ -435,8 +436,11 @@ impl<'s> Tokens<'s> {
         }
     }
 
-    /// Will read the input stream until the argument F evaluates to `true`
+    /// Will read the input stream until `f` evaluates to `true`. When
-    /// for the current character.
+    /// `eat_match` is true, the token for which `f` was true is consumed.
+    /// Returns the string from the index where this was called offset by
+    /// `offset_start` to the end offset by `offset_end`. The end is before or
+    /// after the match depending on `eat_match`.
     fn read_string_until<F>(
         &mut self,
         mut f: F,
@@ -527,8 +531,8 @@ pub fn is_identifier(string: &str) -> bool {
     true
 }
 
-
 #[cfg(test)]
+#[allow(non_snake_case)]
 mod tests {
     use super::super::test::check;
     use super::*;
@@ -549,31 +553,23 @@ mod tests {
         Slash,
     };
 
-    #[allow(non_snake_case)]
-    fn Str(string: &'static str, terminated: bool) -> Token<'static> {
-        Token::ExprStr { string, terminated }
-    }
-
-    #[allow(non_snake_case)]
-    fn Raw(raw: &'static str, terminated: bool) -> Token<'static> {
-        Token::Raw { raw, terminated }
-    }
-
     /// Test whether the given string tokenizes into the given list of tokens.
     macro_rules! t {
         ($mode:expr, $source:expr => [$($tokens:tt)*]) => {
-            let (exp, spans) = spanned![vec $($tokens)*];
+            let (exp, spans) = span_vec![$($tokens)*];
-            let found = Tokens::new(Position::ZERO, $source, $mode).collect::<Vec<_>>();
+            let found = Tokens::new($source, Position::ZERO, $mode).collect::<Vec<_>>();
             check($source, exp, found, spans);
         }
     }
 
-    /// Write down a function token compactly.
+    fn Str(string: &str, terminated: bool) -> Token { Token::ExprStr { string, terminated } }
+    fn Raw(raw: &str, terminated: bool) -> Token { Token::Raw { raw, terminated } }
+
     macro_rules! func {
         ($header:expr, Some($($tokens:tt)*), $terminated:expr) => {
             Function {
                 header: $header,
-                body: Some(spanned![item $($tokens)*]),
+                body: Some(span_item!(($($tokens)*))),
                 terminated: $terminated,
             }
         };
@@ -674,12 +670,12 @@ mod tests {
     fn tokenize_functions() {
         t!(Body, "a[f]"           => [T("a"), func!("f", None, true)]);
         t!(Body, "[f]a"           => [func!("f", None, true), T("a")]);
-        t!(Body, "\n\n[f][ ]"     => [S(2), func!("f", Some((0:4, 0:5, " ")), true)]);
+        t!(Body, "\n\n[f][ ]"     => [S(2), func!("f", Some(0:4, 0:5, " "), true)]);
-        t!(Body, "abc [f][ ]a"    => [T("abc"), S(0), func!("f", Some((0:4, 0:5, " ")), true), T("a")]);
+        t!(Body, "abc [f][ ]a"    => [T("abc"), S(0), func!("f", Some(0:4, 0:5, " "), true), T("a")]);
         t!(Body, "[f: [=][*]]"    => [func!("f: [=][*]", None, true)]);
-        t!(Body, "[_][[,],],"     => [func!("_", Some((0:4, 0:8, "[,],")), true), T(",")]);
+        t!(Body, "[_][[,],],"     => [func!("_", Some(0:4, 0:8, "[,],"), true), T(",")]);
-        t!(Body, "[=][=][=]"      => [func!("=", Some((0:4, 0:5, "=")), true), func!("=", None, true)]);
+        t!(Body, "[=][=][=]"      => [func!("=", Some(0:4, 0:5, "="), true), func!("=", None, true)]);
-        t!(Body, "[=][[=][=][=]]" => [func!("=", Some((0:4, 0:13, "[=][=][=]")), true)]);
+        t!(Body, "[=][[=][=][=]]" => [func!("=", Some(0:4, 0:13, "[=][=][=]"), true)]);
         t!(Header, "["            => [func!("", None, false)]);
         t!(Header, "]"            => [Invalid("]")]);
     }
@@ -693,25 +689,25 @@ mod tests {
         t!(Body, "[f: `]"         => [func!("f: `", None, true)]);
 
         // End of function with strings and carets in bodies
-        t!(Body, "[f][\"]"        => [func!("f", Some((0:4, 0:5, "\"")), true)]);
+        t!(Body, "[f][\"]"        => [func!("f", Some(0:4, 0:5, "\""), true)]);
-        t!(Body, r#"[f][\"]"#     => [func!("f", Some((0:4, 0:6, r#"\""#)), true)]);
+        t!(Body, r#"[f][\"]"#     => [func!("f", Some(0:4, 0:6, r#"\""#), true)]);
-        t!(Body, "[f][`]"         => [func!("f", Some((0:4, 0:6, "`]")), false)]);
+        t!(Body, "[f][`]"         => [func!("f", Some(0:4, 0:6, "`]"), false)]);
-        t!(Body, "[f][\\`]"       => [func!("f", Some((0:4, 0:6, "\\`")), true)]);
+        t!(Body, "[f][\\`]"       => [func!("f", Some(0:4, 0:6, "\\`"), true)]);
-        t!(Body, "[f][`raw`]"     => [func!("f", Some((0:4, 0:9, "`raw`")), true)]);
+        t!(Body, "[f][`raw`]"     => [func!("f", Some(0:4, 0:9, "`raw`"), true)]);
-        t!(Body, "[f][`raw]"      => [func!("f", Some((0:4, 0:9, "`raw]")), false)]);
+        t!(Body, "[f][`raw]"      => [func!("f", Some(0:4, 0:9, "`raw]"), false)]);
-        t!(Body, "[f][`raw]`]"    => [func!("f", Some((0:4, 0:10, "`raw]`")), true)]);
+        t!(Body, "[f][`raw]`]"    => [func!("f", Some(0:4, 0:10, "`raw]`"), true)]);
-        t!(Body, "[f][`\\`]"      => [func!("f", Some((0:4, 0:8, "`\\`]")), false)]);
+        t!(Body, "[f][`\\`]"      => [func!("f", Some(0:4, 0:8, "`\\`]"), false)]);
-        t!(Body, "[f][`\\\\`]"    => [func!("f", Some((0:4, 0:8, "`\\\\`")), true)]);
+        t!(Body, "[f][`\\\\`]"    => [func!("f", Some(0:4, 0:8, "`\\\\`"), true)]);
 
         // End of function with comments
-        t!(Body, "[f][/*]"        => [func!("f", Some((0:4, 0:7, "/*]")), false)]);
+        t!(Body, "[f][/*]"        => [func!("f", Some(0:4, 0:7, "/*]"), false)]);
-        t!(Body, "[f][/*`*/]"     => [func!("f", Some((0:4, 0:9, "/*`*/")), true)]);
+        t!(Body, "[f][/*`*/]"     => [func!("f", Some(0:4, 0:9, "/*`*/"), true)]);
         t!(Body, "[f: //]\n]"     => [func!("f: //]\n", None, true)]);
         t!(Body, "[f: \"//]\n]"   => [func!("f: \"//]\n]", None, false)]);
 
         // End of function with escaped brackets
-        t!(Body, "[f][\\]]"       => [func!("f", Some((0:4, 0:6, "\\]")), true)]);
+        t!(Body, "[f][\\]]"       => [func!("f", Some(0:4, 0:6, "\\]"), true)]);
-        t!(Body, "[f][\\[]"       => [func!("f", Some((0:4, 0:6, "\\[")), true)]);
+        t!(Body, "[f][\\[]"       => [func!("f", Some(0:4, 0:6, "\\["), true)]);
     }
 
     #[test]