From b4be25e43b1ee9da924d13b7f2e8289f12bd2c3b Mon Sep 17 00:00:00 2001 From: Laurenz Date: Thu, 24 Oct 2019 19:10:03 +0200 Subject: [PATCH] =?UTF-8?q?Prettify=20peeking=20and=20rearrange=20syntax/p?= =?UTF-8?q?arsing=20modules=20=F0=9F=A7=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/func/helpers.rs | 11 ++-- src/func/mod.rs | 8 ++- src/lib.rs | 4 +- src/{syntax.rs => syntax/mod.rs} | 6 +++ src/{parsing/mod.rs => syntax/parsing.rs} | 19 +++---- src/{parsing => syntax}/tokens.rs | 61 ++++++++--------------- 6 files changed, 46 insertions(+), 63 deletions(-) rename src/{syntax.rs => syntax/mod.rs} (96%) rename src/{parsing/mod.rs => syntax/parsing.rs} (98%) rename src/{parsing => syntax}/tokens.rs (92%) diff --git a/src/func/helpers.rs b/src/func/helpers.rs index d562284f8..ea0f6b2f0 100644 --- a/src/func/helpers.rs +++ b/src/func/helpers.rs @@ -1,11 +1,14 @@ -use super::prelude::*; use std::iter::Peekable; use std::slice::Iter; +use super::prelude::*; /// Implement the function trait more concisely. #[macro_export] macro_rules! function { (data: $ident:ident, $($tts:tt)*) => { + #[allow(unused_imports)] + use $crate::func::prelude::*; + impl Function for $ident { function!(@parse $ident, $($tts)*); } @@ -64,7 +67,7 @@ macro_rules! parse { (optional: $body:expr, $ctx:expr) => { if let Some(body) = $body { - Some($crate::parsing::parse(body, $ctx)?) + Some($crate::syntax::parse(body, $ctx)?) } else { None } @@ -72,7 +75,7 @@ macro_rules! parse { (required: $body:expr, $ctx:expr) => { if let Some(body) = $body { - $crate::parsing::parse(body, $ctx)? + $crate::syntax::parse(body, $ctx)? } else { err!("expected body"); } @@ -83,7 +86,7 @@ macro_rules! parse { #[macro_export] macro_rules! err { ($($tts:tt)*) => { - return Err(ParseError::new(format!($($tts)*))); + return Err($crate::syntax::ParseError::new(format!($($tts)*))); }; } diff --git a/src/func/mod.rs b/src/func/mod.rs index 9a6fcbd14..b3918253a 100644 --- a/src/func/mod.rs +++ b/src/func/mod.rs @@ -4,10 +4,7 @@ use std::any::Any; use std::collections::HashMap; use std::fmt::{self, Debug, Formatter}; -use crate::layout::{Layout, LayoutContext, Alignment, LayoutResult, MultiLayout}; -use crate::parsing::{ParseContext, ParseResult}; -use crate::style::TextStyle; -use crate::syntax::{FuncHeader, SyntaxTree}; +use self::prelude::*; #[macro_use] mod helpers; @@ -18,9 +15,10 @@ pub mod prelude { pub use crate::func::{Command, CommandList, Function}; pub use crate::layout::{layout_tree, Layout, LayoutContext, MultiLayout}; pub use crate::layout::{Flow, Alignment, LayoutError, LayoutResult}; - pub use crate::parsing::{parse, ParseContext, ParseError, ParseResult}; pub use crate::syntax::{Expression, FuncHeader, SyntaxTree}; + pub use crate::syntax::{parse, ParseContext, ParseError, ParseResult}; pub use crate::size::{Size, Size2D, SizeBox}; + pub use crate::style::{PageStyle, TextStyle}; pub use super::helpers::*; } diff --git a/src/lib.rs b/src/lib.rs index 0b559be99..9259ffb75 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -22,9 +22,8 @@ use toddle::query::{FontLoader, FontProvider, SharedFontLoader}; use crate::func::Scope; use crate::layout::{layout_tree, LayoutContext, MultiLayout}; use crate::layout::{Alignment, Flow, LayoutError, LayoutResult, LayoutSpace}; -use crate::parsing::{parse, ParseContext, ParseError, ParseResult}; +use crate::syntax::{SyntaxTree, parse, ParseContext, ParseError, ParseResult}; use crate::style::{PageStyle, TextStyle}; -use crate::syntax::SyntaxTree; #[macro_use] mod macros; @@ -33,7 +32,6 @@ pub mod export; pub mod func; pub mod layout; pub mod library; -pub mod parsing; pub mod size; pub mod style; pub mod syntax; diff --git a/src/syntax.rs b/src/syntax/mod.rs similarity index 96% rename from src/syntax.rs rename to src/syntax/mod.rs index 9c7e4908b..b46beb36a 100644 --- a/src/syntax.rs +++ b/src/syntax/mod.rs @@ -6,6 +6,12 @@ use std::fmt::{self, Display, Formatter}; use crate::func::Function; use crate::size::Size; +mod tokens; +mod parsing; + +pub use tokens::{tokenize, Tokens}; +pub use parsing::{parse, ParseContext, ParseError, ParseResult}; + /// A logical unit of the incoming text stream. #[derive(Debug, Copy, Clone, Eq, PartialEq)] pub enum Token<'s> { diff --git a/src/parsing/mod.rs b/src/syntax/parsing.rs similarity index 98% rename from src/parsing/mod.rs rename to src/syntax/parsing.rs index 369205957..c22870854 100644 --- a/src/parsing/mod.rs +++ b/src/syntax/parsing.rs @@ -1,16 +1,11 @@ //! Parsing of source code into token streams and syntax trees. use std::collections::HashMap; - use unicode_xid::UnicodeXID; use crate::func::{Function, Scope}; use crate::size::Size; -use crate::syntax::*; - -mod tokens; - -pub use tokens::{tokenize, Tokens}; +use super::*; /// Parses source code into a syntax tree given a context. #[inline] @@ -222,7 +217,7 @@ impl<'s> Parser<'s> { // Find out the string which makes the body of this function. let (start, end) = self .tokens - .current_index() + .string_index() .and_then(|index| { find_closing_bracket(&self.src[index..]).map(|end| (index, index + end)) }) @@ -233,7 +228,7 @@ impl<'s> Parser<'s> { let body = parser(&header, Some(body_string), self.ctx)?; // Skip to the end of the function in the token stream. - self.tokens.goto(end); + self.tokens.set_string_index(end); // Now the body should be closed. assert!(self.tokens.next() == Some(Token::RightBracket)); @@ -377,13 +372,13 @@ impl<'s> PeekableTokens<'s> { } /// The index of the first character of the next token in the source string. - fn current_index(&mut self) -> Option { - self.tokens.chars.current_index() + fn string_index(&mut self) -> Option { + self.tokens.chars.string_index() } /// Go to a new position in the underlying string. - fn goto(&mut self, index: usize) { - self.tokens.chars.goto(index); + fn set_string_index(&mut self, index: usize) { + self.tokens.chars.set_string_index(index); self.peeked = None; } } diff --git a/src/parsing/tokens.rs b/src/syntax/tokens.rs similarity index 92% rename from src/parsing/tokens.rs rename to src/syntax/tokens.rs index 295b13d86..460884a4b 100644 --- a/src/parsing/tokens.rs +++ b/src/syntax/tokens.rs @@ -1,8 +1,6 @@ use std::str::CharIndices; - use smallvec::SmallVec; - -use crate::syntax::*; +use super::*; /// Builds an iterator over the tokens of the source code. #[inline] @@ -223,7 +221,7 @@ impl<'s> Iterator for Tokens<'s> { // Find out when the word ends. let mut end = (next_pos, next); while let Some((index, c)) = self.chars.peek() { - let second = self.chars.peek_second().map(|p| p.1); + let second = self.chars.peekn(1).map(|p| p.1); // Whether the next token is still from the text or not. let continues = match c { @@ -266,66 +264,53 @@ fn is_newline_char(character: char) -> bool { /// A (index, char) iterator with double lookahead. #[derive(Debug, Clone)] pub struct PeekableChars<'s> { - offset: usize, string: &'s str, chars: CharIndices<'s>, - peek1: Option>, - peek2: Option>, + base: usize, + peeked: SmallVec<[Option<(usize, char)>; 2]>, } impl<'s> PeekableChars<'s> { /// Create a new iterator from a string. pub fn new(string: &'s str) -> PeekableChars<'s> { PeekableChars { - offset: 0, string, chars: string.char_indices(), - peek1: None, - peek2: None, + base: 0, + peeked: SmallVec::new(), } } /// Peek at the next element. pub fn peek(&mut self) -> Option<(usize, char)> { - match self.peek1 { - Some(peeked) => peeked, - None => { - let next = self.next_inner(); - self.peek1 = Some(next); - next - } - } + self.peekn(0) } /// Peek at the element after the next element. - pub fn peek_second(&mut self) -> Option<(usize, char)> { - match self.peek2 { - Some(peeked) => peeked, - None => { - self.peek(); - let next = self.next_inner(); - self.peek2 = Some(next); - next - } + pub fn peekn(&mut self, n: usize) -> Option<(usize, char)> { + while self.peeked.len() <= n { + let next = self.next_inner(); + self.peeked.push(next); } + + self.peeked[n] } /// Return the next value of the inner iterator mapped with the offset. pub fn next_inner(&mut self) -> Option<(usize, char)> { - self.chars.next().map(|(i, c)| (i + self.offset, c)) + self.chars.next().map(|(i, c)| (i + self.base, c)) } /// The index of the first character of the next token in the source string. - pub fn current_index(&mut self) -> Option { + pub fn string_index(&mut self) -> Option { self.peek().map(|p| p.0) } /// Go to a new position in the underlying string. - pub fn goto(&mut self, index: usize) { - self.offset = index; + pub fn set_string_index(&mut self, index: usize) { self.chars = self.string[index..].char_indices(); - self.peek1 = None; - self.peek2 = None; + self.base = index; + self.peeked.clear(); } } @@ -333,12 +318,10 @@ impl Iterator for PeekableChars<'_> { type Item = (usize, char); fn next(&mut self) -> Option<(usize, char)> { - match self.peek1.take() { - Some(value) => { - self.peek1 = self.peek2.take(); - value - } - None => self.next_inner(), + if !self.peeked.is_empty() { + self.peeked.remove(0) + } else { + self.next_inner() } } }