Prettify peeking and rearrange syntax/parsing modules 🧶

src/func/helpers.rs

@@ -1,11 +1,14 @@
-use super::prelude::*;
 use std::iter::Peekable;
 use std::slice::Iter;
+use super::prelude::*;
 
 /// Implement the function trait more concisely.
 #[macro_export]
 macro_rules! function {
     (data: $ident:ident, $($tts:tt)*) => {
+        #[allow(unused_imports)]
+        use $crate::func::prelude::*;
+
         impl Function for $ident {
             function!(@parse $ident, $($tts)*);
         }
@@ -64,7 +67,7 @@ macro_rules! parse {
 
     (optional: $body:expr, $ctx:expr) => {
         if let Some(body) = $body {
-            Some($crate::parsing::parse(body, $ctx)?)
+            Some($crate::syntax::parse(body, $ctx)?)
         } else {
             None
         }
@@ -72,7 +75,7 @@ macro_rules! parse {
 
     (required: $body:expr, $ctx:expr) => {
         if let Some(body) = $body {
-            $crate::parsing::parse(body, $ctx)?
+            $crate::syntax::parse(body, $ctx)?
         } else {
             err!("expected body");
         }
@@ -83,7 +86,7 @@ macro_rules! parse {
 #[macro_export]
 macro_rules! err {
     ($($tts:tt)*) => {
-        return Err(ParseError::new(format!($($tts)*)));
+        return Err($crate::syntax::ParseError::new(format!($($tts)*)));
     };
 }
 

src/func/mod.rs

@@ -4,10 +4,7 @@ use std::any::Any;
 use std::collections::HashMap;
 use std::fmt::{self, Debug, Formatter};
 
-use crate::layout::{Layout, LayoutContext, Alignment, LayoutResult, MultiLayout};
+use self::prelude::*;
-use crate::parsing::{ParseContext, ParseResult};
-use crate::style::TextStyle;
-use crate::syntax::{FuncHeader, SyntaxTree};
 
 #[macro_use]
 mod helpers;
@@ -18,9 +15,10 @@ pub mod prelude {
     pub use crate::func::{Command, CommandList, Function};
     pub use crate::layout::{layout_tree, Layout, LayoutContext, MultiLayout};
     pub use crate::layout::{Flow, Alignment, LayoutError, LayoutResult};
-    pub use crate::parsing::{parse, ParseContext, ParseError, ParseResult};
     pub use crate::syntax::{Expression, FuncHeader, SyntaxTree};
+    pub use crate::syntax::{parse, ParseContext, ParseError, ParseResult};
     pub use crate::size::{Size, Size2D, SizeBox};
+    pub use crate::style::{PageStyle, TextStyle};
     pub use super::helpers::*;
 }
 

src/lib.rs

@@ -22,9 +22,8 @@ use toddle::query::{FontLoader, FontProvider, SharedFontLoader};
 use crate::func::Scope;
 use crate::layout::{layout_tree, LayoutContext, MultiLayout};
 use crate::layout::{Alignment, Flow, LayoutError, LayoutResult, LayoutSpace};
-use crate::parsing::{parse, ParseContext, ParseError, ParseResult};
+use crate::syntax::{SyntaxTree, parse, ParseContext, ParseError, ParseResult};
 use crate::style::{PageStyle, TextStyle};
-use crate::syntax::SyntaxTree;
 
 #[macro_use]
 mod macros;
@@ -33,7 +32,6 @@ pub mod export;
 pub mod func;
 pub mod layout;
 pub mod library;
-pub mod parsing;
 pub mod size;
 pub mod style;
 pub mod syntax;

src/syntax/mod.rs

@@ -6,6 +6,12 @@ use std::fmt::{self, Display, Formatter};
 use crate::func::Function;
 use crate::size::Size;
 
+mod tokens;
+mod parsing;
+
+pub use tokens::{tokenize, Tokens};
+pub use parsing::{parse, ParseContext, ParseError, ParseResult};
+
 /// A logical unit of the incoming text stream.
 #[derive(Debug, Copy, Clone, Eq, PartialEq)]
 pub enum Token<'s> {

src/syntax/parsing.rs

@@ -1,16 +1,11 @@
 //! Parsing of source code into token streams and syntax trees.
 
 use std::collections::HashMap;
-
 use unicode_xid::UnicodeXID;
 
 use crate::func::{Function, Scope};
 use crate::size::Size;
-use crate::syntax::*;
+use super::*;
-
-mod tokens;
-
-pub use tokens::{tokenize, Tokens};
 
 /// Parses source code into a syntax tree given a context.
 #[inline]
@@ -222,7 +217,7 @@ impl<'s> Parser<'s> {
             // Find out the string which makes the body of this function.
             let (start, end) = self
                 .tokens
-                .current_index()
+                .string_index()
                 .and_then(|index| {
                     find_closing_bracket(&self.src[index..]).map(|end| (index, index + end))
                 })
@@ -233,7 +228,7 @@ impl<'s> Parser<'s> {
             let body = parser(&header, Some(body_string), self.ctx)?;
 
             // Skip to the end of the function in the token stream.
-            self.tokens.goto(end);
+            self.tokens.set_string_index(end);
 
             // Now the body should be closed.
             assert!(self.tokens.next() == Some(Token::RightBracket));
@@ -377,13 +372,13 @@ impl<'s> PeekableTokens<'s> {
     }
 
     /// The index of the first character of the next token in the source string.
-    fn current_index(&mut self) -> Option<usize> {
+    fn string_index(&mut self) -> Option<usize> {
-        self.tokens.chars.current_index()
+        self.tokens.chars.string_index()
     }
 
     /// Go to a new position in the underlying string.
-    fn goto(&mut self, index: usize) {
+    fn set_string_index(&mut self, index: usize) {
-        self.tokens.chars.goto(index);
+        self.tokens.chars.set_string_index(index);
         self.peeked = None;
     }
 }

src/syntax/tokens.rs

@@ -1,8 +1,6 @@
 use std::str::CharIndices;
-
 use smallvec::SmallVec;
-
+use super::*;
-use crate::syntax::*;
 
 /// Builds an iterator over the tokens of the source code.
 #[inline]
@@ -223,7 +221,7 @@ impl<'s> Iterator for Tokens<'s> {
                 // Find out when the word ends.
                 let mut end = (next_pos, next);
                 while let Some((index, c)) = self.chars.peek() {
-                    let second = self.chars.peek_second().map(|p| p.1);
+                    let second = self.chars.peekn(1).map(|p| p.1);
 
                     // Whether the next token is still from the text or not.
                     let continues = match c {
@@ -266,66 +264,53 @@ fn is_newline_char(character: char) -> bool {
 /// A (index, char) iterator with double lookahead.
 #[derive(Debug, Clone)]
 pub struct PeekableChars<'s> {
-    offset: usize,
     string: &'s str,
     chars: CharIndices<'s>,
-    peek1: Option<Option<(usize, char)>>,
+    base: usize,
-    peek2: Option<Option<(usize, char)>>,
+    peeked: SmallVec<[Option<(usize, char)>; 2]>,
 }
 
 impl<'s> PeekableChars<'s> {
     /// Create a new iterator from a string.
     pub fn new(string: &'s str) -> PeekableChars<'s> {
         PeekableChars {
-            offset: 0,
             string,
             chars: string.char_indices(),
-            peek1: None,
+            base: 0,
-            peek2: None,
+            peeked: SmallVec::new(),
         }
     }
 
     /// Peek at the next element.
     pub fn peek(&mut self) -> Option<(usize, char)> {
-        match self.peek1 {
+        self.peekn(0)
-            Some(peeked) => peeked,
-            None => {
-                let next = self.next_inner();
-                self.peek1 = Some(next);
-                next
-            }
-        }
     }
 
     /// Peek at the element after the next element.
-    pub fn peek_second(&mut self) -> Option<(usize, char)> {
+    pub fn peekn(&mut self, n: usize) -> Option<(usize, char)> {
-        match self.peek2 {
+        while self.peeked.len() <= n {
-            Some(peeked) => peeked,
-            None => {
-                self.peek();
             let next = self.next_inner();
-                self.peek2 = Some(next);
+            self.peeked.push(next);
-                next
-            }
         }
+
+        self.peeked[n]
     }
 
     /// Return the next value of the inner iterator mapped with the offset.
     pub fn next_inner(&mut self) -> Option<(usize, char)> {
-        self.chars.next().map(|(i, c)| (i + self.offset, c))
+        self.chars.next().map(|(i, c)| (i + self.base, c))
     }
 
     /// The index of the first character of the next token in the source string.
-    pub fn current_index(&mut self) -> Option<usize> {
+    pub fn string_index(&mut self) -> Option<usize> {
         self.peek().map(|p| p.0)
     }
 
     /// Go to a new position in the underlying string.
-    pub fn goto(&mut self, index: usize) {
+    pub fn set_string_index(&mut self, index: usize) {
-        self.offset = index;
         self.chars = self.string[index..].char_indices();
-        self.peek1 = None;
+        self.base = index;
-        self.peek2 = None;
+        self.peeked.clear();
     }
 }
 
@@ -333,12 +318,10 @@ impl Iterator for PeekableChars<'_> {
     type Item = (usize, char);
 
     fn next(&mut self) -> Option<(usize, char)> {
-        match self.peek1.take() {
+        if !self.peeked.is_empty() {
-            Some(value) => {
+            self.peeked.remove(0)
-                self.peek1 = self.peek2.take();
+        } else {
-                value
+            self.next_inner()
-            }
-            None => self.next_inner(),
         }
     }
 }