From 08b91a265fcda74f5463473938ec33873b49a7f7 Mon Sep 17 00:00:00 2001 From: Laurenz Date: Thu, 16 Jan 2020 17:51:04 +0100 Subject: [PATCH] =?UTF-8?q?Powerful=20parser=20testing=20=F0=9F=90=B1?= =?UTF-8?q?=E2=80=8D=F0=9F=91=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.toml | 10 +- build.rs | 11 +- src/func/macros.rs | 41 ++-- src/func/mod.rs | 11 +- src/library/align.rs | 4 +- src/library/boxed.rs | 6 +- src/library/direction.rs | 4 +- src/library/mod.rs | 40 ++-- src/syntax/color.rs | 3 - src/syntax/expr.rs | 4 + src/syntax/mod.rs | 113 ++++++---- src/syntax/parsing.rs | 27 +-- src/syntax/span.rs | 18 +- tests/{layouts => layouter}/coma.typ | 0 tests/{layouts => layouter}/stack.typ | 0 tests/parse.rs | 236 ------------------- tests/{parsing => parser}/tokens.rs | 8 +- tests/parser/trees.rs | 33 +++ tests/parsing/trees.rs | 20 -- tests/{layout.rs => src/layouter.rs} | 9 +- tests/src/parser.rs | 311 ++++++++++++++++++++++++++ tests/{ => src}/render.py | 14 +- tests/src/spanless.rs | 62 +++++ 23 files changed, 582 insertions(+), 403 deletions(-) delete mode 100644 src/syntax/color.rs rename tests/{layouts => layouter}/coma.typ (100%) rename tests/{layouts => layouter}/stack.typ (100%) delete mode 100644 tests/parse.rs rename tests/{parsing => parser}/tokens.rs (92%) create mode 100644 tests/parser/trees.rs delete mode 100644 tests/parsing/trees.rs rename tests/{layout.rs => src/layouter.rs} (96%) create mode 100644 tests/src/parser.rs rename tests/{ => src}/render.py (93%) create mode 100644 tests/src/spanless.rs diff --git a/Cargo.toml b/Cargo.toml index b2c385d41..908503bf0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,17 +19,17 @@ default = ["fs-provider", "futures-executor"] fs-provider = ["toddle/fs-provider"] [[bin]] -name = "typst-bin" +name = "typst" path = "src/bin/main.rs" required-features = ["futures-executor"] [[test]] -name = "layout" -path = "tests/layout.rs" +name = "layouter" +path = "tests/src/layouter.rs" harness = false required-features = ["futures-executor"] [[test]] -name = "parse" -path = "tests/parse.rs" +name = "parser" +path = "tests/src/parser.rs" harness = false diff --git a/build.rs b/build.rs index 0c3f1da51..e16707555 100644 --- a/build.rs +++ b/build.rs @@ -1,19 +1,20 @@ use std::fs::{self, create_dir_all, read_dir, read_to_string}; use std::ffi::OsStr; + fn main() -> Result<(), Box> { create_dir_all("tests/cache")?; // Make sure the script reruns if this file changes or files are // added/deleted in the parsing folder. println!("cargo:rerun-if-changed=build.rs"); - println!("cargo:rerun-if-changed=tests/cache/parse"); - println!("cargo:rerun-if-changed=tests/parsing"); + println!("cargo:rerun-if-changed=tests/cache/parser-tests.rs"); + println!("cargo:rerun-if-changed=tests/parser"); // Compile all parser tests into a single giant vector. let mut code = "vec![".to_string(); - for entry in read_dir("tests/parsing")? { + for entry in read_dir("tests/parser")? { let path = entry?.path(); if path.extension() != Some(OsStr::new("rs")) { continue; @@ -25,7 +26,7 @@ fn main() -> Result<(), Box> { // Make sure this also reruns if the contents of a file in parsing // change. This is not ensured by rerunning only on the folder. - println!("cargo:rerun-if-changed=tests/parsing/{}.rs", name); + println!("cargo:rerun-if-changed=tests/parser/{}.rs", name); code.push_str(&format!("(\"{}\", tokens!{{", name)); @@ -44,7 +45,7 @@ fn main() -> Result<(), Box> { code.push(']'); - fs::write("tests/cache/parse", code)?; + fs::write("tests/cache/parser-tests.rs", code)?; Ok(()) } diff --git a/src/func/macros.rs b/src/func/macros.rs index 1083e53c6..90c3b11ef 100644 --- a/src/func/macros.rs +++ b/src/func/macros.rs @@ -52,46 +52,43 @@ macro_rules! function { }; // (1-arg) Parse a parse-definition with only the first argument. - (@parse $type:ident $meta:ty | parse($args:ident) $code:block $($rest:tt)*) => { - function!(@parse $type $meta | parse($args, _body, _ctx, _meta) $code $($rest)*); + (@parse $type:ident $meta:ty | parse($header:ident) $code:block $($rest:tt)*) => { + function!(@parse $type $meta | parse($header, _body, _ctx, _meta) $code $($rest)*); }; // (2-arg) Parse a parse-definition with only the first two arguments. (@parse $type:ident $meta:ty | - parse($args:ident, $body:pat) $code:block $($rest:tt)* + parse($header:ident, $body:pat) $code:block $($rest:tt)* ) => { - function!(@parse $type $meta | parse($args, $body, _ctx, _meta) $code $($rest)*); + function!(@parse $type $meta | parse($header, $body, _ctx, _meta) $code $($rest)*); }; // (3-arg) Parse a parse-definition with only the first three arguments. (@parse $type:ident $meta:ty | - parse($args:ident, $body:pat, $ctx:pat) $code:block $($rest:tt)* + parse($header:ident, $body:pat, $ctx:pat) $code:block $($rest:tt)* ) => { - function!(@parse $type $meta | parse($args, $body, $ctx, _meta) $code $($rest)*); + function!(@parse $type $meta | parse($header, $body, $ctx, _meta) $code $($rest)*); }; // (4-arg) Parse a parse-definition with all four arguments. (@parse $type:ident $meta:ty | - parse($args:ident, $body:pat, $ctx:pat, $metadata:pat) $code:block + parse($header:ident, $body:pat, $ctx:pat, $metadata:pat) $code:block $($rest:tt)* ) => { - use $crate::func::prelude::*; - impl $crate::func::ParseFunc for $type { type Meta = $meta; fn parse( - args: FuncArgs, + header: $crate::syntax::FuncHeader, $body: Option<&str>, - $ctx: ParseContext, + $ctx: $crate::syntax::ParseContext, $metadata: Self::Meta, - ) -> ParseResult where Self: Sized { + ) -> $crate::syntax::ParseResult where Self: Sized { #[allow(unused_mut)] - let mut $args = args; + let mut $header = header; let val = $code; - if !$args.is_empty() { - return Err($crate::TypesetError - ::with_message("unexpected arguments")); + if !$header.args.is_empty() { + return Err($crate::TypesetError::with_message("unexpected arguments")); } Ok(val) } @@ -112,14 +109,14 @@ macro_rules! function { // (2-arg) Parse a layout-definition with all arguments. (@layout $type:ident | layout($this:ident, $ctx:pat) $code:block) => { - use $crate::func::prelude::*; - - impl LayoutFunc for $type { + impl $crate::func::LayoutFunc for $type { fn layout<'a, 'life0, 'life1, 'async_trait>( &'a $this, - $ctx: LayoutContext<'life0, 'life1> - ) -> std::pin::Pin>> + 'async_trait + $ctx: $crate::layout::LayoutContext<'life0, 'life1> + ) -> std::pin::Pin> + > + 'async_trait >> where 'a: 'async_trait, diff --git a/src/func/mod.rs b/src/func/mod.rs index 90b2a31d3..bfc2774c6 100644 --- a/src/func/mod.rs +++ b/src/func/mod.rs @@ -14,12 +14,7 @@ mod macros; pub mod prelude { pub use crate::func::{Scope, ParseFunc, LayoutFunc, Command, Commands}; pub use crate::layout::prelude::*; - pub use crate::syntax::{ - ParseContext, ParseResult, - SyntaxTree, FuncCall, FuncArgs, - Expression, Ident, ExpressionKind, - Spanned, Span - }; + pub use crate::syntax::*; pub use crate::size::{Size, Size2D, SizeBox, ValueBox, ScaleSize, FSize, PSize}; pub use crate::style::{LayoutStyle, PageStyle, TextStyle}; pub use Command::*; @@ -31,7 +26,7 @@ pub trait ParseFunc { /// Parse the header and body into this function given a context. fn parse( - args: FuncArgs, + header: FuncHeader, body: Option<&str>, ctx: ParseContext, metadata: Self::Meta, @@ -125,7 +120,7 @@ pub struct Scope { /// A function which parses the source of a function into a function type which /// implements [`LayoutFunc`]. type Parser = dyn Fn( - FuncArgs, + FuncHeader, Option<&str>, ParseContext ) -> ParseResult>; diff --git a/src/library/align.rs b/src/library/align.rs index 6114c3a37..ca2c787b9 100644 --- a/src/library/align.rs +++ b/src/library/align.rs @@ -10,10 +10,10 @@ function! { map: PosAxisMap, } - parse(args, body, ctx) { + parse(header, body, ctx) { AlignFunc { body: parse!(optional: body, ctx), - map: PosAxisMap::new(&mut args)?, + map: PosAxisMap::new(&mut header.args)?, } } diff --git a/src/library/boxed.rs b/src/library/boxed.rs index da06a371e..af236da4b 100644 --- a/src/library/boxed.rs +++ b/src/library/boxed.rs @@ -13,11 +13,11 @@ function! { debug: Option, } - parse(args, body, ctx) { + parse(header, body, ctx) { BoxFunc { body: parse!(optional: body, ctx).unwrap_or(SyntaxTree::new()), - map: ExtentMap::new(&mut args, false)?, - debug: args.get_key_opt::("debug")?, + map: ExtentMap::new(&mut header.args, false)?, + debug: header.args.get_key_opt::("debug")?, } } diff --git a/src/library/direction.rs b/src/library/direction.rs index 39ac2ccd1..b7a6e212f 100644 --- a/src/library/direction.rs +++ b/src/library/direction.rs @@ -10,10 +10,10 @@ function! { map: PosAxisMap, } - parse(args, body, ctx) { + parse(header, body, ctx) { DirectionFunc { body: parse!(optional: body, ctx), - map: PosAxisMap::new(&mut args)?, + map: PosAxisMap::new(&mut header.args)?, } } diff --git a/src/library/mod.rs b/src/library/mod.rs index f86259048..d91f1b35b 100644 --- a/src/library/mod.rs +++ b/src/library/mod.rs @@ -58,11 +58,11 @@ function! { list: Vec, } - parse(args, body, ctx) { + parse(header, body, ctx) { FontFamilyFunc { body: parse!(optional: body, ctx), list: { - args.iter_pos().map(|arg| match arg.v { + header.args.iter_pos().map(|arg| match arg.v { Expression::Str(s) | Expression::Ident(Ident(s)) => Ok(s.to_lowercase()), _ => error!("expected identifier or string"), @@ -86,11 +86,11 @@ function! { style: FontStyle, } - parse(args, body, ctx) { + parse(header, body, ctx) { FontStyleFunc { body: parse!(optional: body, ctx), style: { - let s = args.get_pos::()?; + let s = header.args.get_pos::()?; match FontStyle::from_str(&s) { Some(style) => style, None => error!("invalid font style: `{}`", s), @@ -114,10 +114,10 @@ function! { weight: FontWeight, } - parse(args, body, ctx) { + parse(header, body, ctx) { FontWeightFunc { body: parse!(optional: body, ctx), - weight: match args.get_pos::()? { + weight: match header.args.get_pos::()? { Expression::Number(weight) => { let weight = weight.round() as i16; FontWeight( @@ -152,10 +152,10 @@ function! { size: ScaleSize, } - parse(args, body, ctx) { + parse(header, body, ctx) { FontSizeFunc { body: parse!(optional: body, ctx), - size: args.get_pos::()?, + size: header.args.get_pos::()?, } } @@ -187,11 +187,11 @@ function! { type Meta = ContentKind; - parse(args, body, ctx, meta) { + parse(header, body, ctx, meta) { ContentSpacingFunc { body: parse!(optional: body, ctx), content: meta, - spacing: args.get_pos::()? as f32, + spacing: header.args.get_pos::()? as f32, } } @@ -256,16 +256,18 @@ function! { type Meta = Option; - parse(args, body, _, meta) { + parse(header, body, _, meta) { parse!(forbidden: body); if let Some(axis) = meta { SpacingFunc { axis: AxisKey::Specific(axis), - spacing: FSize::from_expr(args.get_pos::>()?)?, + spacing: FSize::from_expr( + header.args.get_pos::>()? + )?, } } else { - for arg in args.iter_keys() { + for arg in header.args.iter_keys() { let axis = AxisKey::from_ident(&arg.key) .map_err(|_| error!(@unexpected_argument))?; @@ -295,16 +297,16 @@ function! { Custom(ExtentMap), } - parse(args, body) { + parse(header, body) { parse!(forbidden: body); - if let Some(name) = args.get_pos_opt::()? { - let flip = args.get_key_opt::("flip")?.unwrap_or(false); + if let Some(name) = header.args.get_pos_opt::()? { + let flip = header.args.get_key_opt::("flip")?.unwrap_or(false); let paper = Paper::from_name(name.as_str()) .ok_or_else(|| error!(@"invalid paper name: `{}`", name))?; PageSizeFunc::Paper(paper, flip) } else { - PageSizeFunc::Custom(ExtentMap::new(&mut args, true)?) + PageSizeFunc::Custom(ExtentMap::new(&mut header.args, true)?) } } @@ -341,10 +343,10 @@ function! { map: PaddingMap, } - parse(args, body) { + parse(header, body) { parse!(forbidden: body); PageMarginsFunc { - map: PaddingMap::new(&mut args)?, + map: PaddingMap::new(&mut header.args)?, } } diff --git a/src/syntax/color.rs b/src/syntax/color.rs deleted file mode 100644 index 65525480c..000000000 --- a/src/syntax/color.rs +++ /dev/null @@ -1,3 +0,0 @@ -use super::*; - - diff --git a/src/syntax/expr.rs b/src/syntax/expr.rs index b06b29c80..c4feea749 100644 --- a/src/syntax/expr.rs +++ b/src/syntax/expr.rs @@ -107,6 +107,10 @@ impl Object { impl Display for Object { fn fmt(&self, f: &mut Formatter) -> fmt::Result { + if self.pairs.len() == 0 { + return write!(f, "{{}}"); + } + write!(f, "{{ ")?; let mut first = true; diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs index 1c72de4de..bcec05af5 100644 --- a/src/syntax/mod.rs +++ b/src/syntax/mod.rs @@ -9,7 +9,6 @@ use crate::size::{Size, ScaleSize}; pub type ParseResult = crate::TypesetResult; -pub_use_mod!(color); pub_use_mod!(expr); pub_use_mod!(tokens); pub_use_mod!(parsing); @@ -93,7 +92,7 @@ impl SyntaxTree { } /// A node in the syntax tree. -#[derive(Debug, PartialEq)] +#[derive(PartialEq)] pub enum Node { /// A number of whitespace characters containing less than two newlines. Space, @@ -111,6 +110,28 @@ pub enum Node { Func(FuncCall), } +impl Display for Node { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + match self { + Node::Space => write!(f, "Space"), + Node::Newline => write!(f, "Newline"), + Node::Text(text) => write!(f, "{:?}", text), + Node::ToggleItalic => write!(f, "ToggleItalic"), + Node::ToggleBolder => write!(f, "ToggleBold"), + Node::ToggleMonospace => write!(f, "ToggleMonospace"), + Node::Func(func) => { + if f.alternate() { + write!(f, "{:#?}", func.0) + } else { + write!(f, "{:?}", func.0) + } + } + } + } +} + +debug_display!(Node); + /// An invocation of a function. #[derive(Debug)] pub struct FuncCall(pub Box); @@ -121,59 +142,20 @@ impl PartialEq for FuncCall { } } -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct Colorization { - pub colors: Vec>, -} - -/// Entities which can be colored by syntax highlighting. -#[derive(Debug, Copy, Clone, Eq, PartialEq)] -pub enum ColorToken { - Comment, - - Bracket, - FuncName, - Colon, - - Key, - Equals, - Comma, - - Paren, - Brace, - - ExprIdent, - ExprStr, - ExprNumber, - ExprSize, - ExprBool, - - Bold, - Italic, - Monospace, - - Invalid, -} - -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct ErrorMap { - pub errors: Vec>, -} - -#[derive(Debug)] +#[derive(Debug, Clone, PartialEq)] pub struct FuncHeader { pub name: Spanned, pub args: FuncArgs, } -#[derive(Debug)] +#[derive(Debug, Clone, PartialEq)] pub struct FuncArgs { - positional: Tuple, - keyword: Object, + pub positional: Tuple, + pub keyword: Object, } impl FuncArgs { - fn new() -> FuncArgs { + pub fn new() -> FuncArgs { FuncArgs { positional: Tuple::new(), keyword: Object::new(), @@ -258,3 +240,42 @@ fn expect(opt: ParseResult>) -> ParseResult { Err(e) => Err(e), } } + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct Colorization { + pub tokens: Vec>, +} + +/// Entities which can be colored by syntax highlighting. +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +pub enum ColorToken { + Comment, + + Bracket, + FuncName, + Colon, + + Key, + Equals, + Comma, + + Paren, + Brace, + + ExprIdent, + ExprStr, + ExprNumber, + ExprSize, + ExprBool, + + Bold, + Italic, + Monospace, + + Invalid, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct ErrorMap { + pub errors: Vec>, +} diff --git a/src/syntax/parsing.rs b/src/syntax/parsing.rs index bf3bea893..f0a686418 100644 --- a/src/syntax/parsing.rs +++ b/src/syntax/parsing.rs @@ -33,7 +33,7 @@ impl<'s> Parser<'s> { src, ctx, error_map: ErrorMap { errors: vec![] }, - colorization: Colorization { colors: vec![] }, + colorization: Colorization { tokens: vec![] }, tokens: Tokens::new(src), peeked: None, @@ -114,8 +114,6 @@ impl<'s> Parser<'s> { } fn parse_func_call(&mut self, header: Option) -> Option { - println!("peek: {:?}", self.peek()); - let body = if self.peek() == Some(LeftBracket) { self.eat(); @@ -140,13 +138,15 @@ impl<'s> Parser<'s> { }; let header = header?; - let name = header.name; - let parser = self.ctx.scope.get_parser(name.v.as_str()).or_else(|| { - self.error(format!("unknown function: `{}`", name.v), name.span); + let parser = self.ctx.scope.get_parser(header.name.v.as_str()).or_else(|| { + self.error( + format!("unknown function: `{}`", header.name.v), + header.name.span + ); None })?; - Some(FuncCall(parser(header.args, body, self.ctx).unwrap())) + Some(FuncCall(parser(header, body, self.ctx).unwrap())) } fn parse_func_name(&mut self) -> Option> { @@ -163,16 +163,17 @@ impl<'s> Parser<'s> { } fn parse_func_args(&mut self) -> FuncArgs { - // unimplemented!() + // todo!() + self.eat_until(|t| t == RightBracket, true); FuncArgs::new() } fn parse_tuple(&mut self) -> Spanned { - unimplemented!("parse_tuple") + todo!("parse_tuple") } fn parse_object(&mut self) -> Spanned { - unimplemented!("parse_object") + todo!("parse_object") } fn skip_whitespace(&mut self) { @@ -207,13 +208,13 @@ impl<'s> Parser<'s> { fn color(&mut self, token: Spanned, replace_last: bool) { if replace_last { - if let Some(last) = self.colorization.colors.last_mut() { + if let Some(last) = self.colorization.tokens.last_mut() { *last = token; return; } } - self.colorization.colors.push(token); + self.colorization.tokens.push(token); } fn color_token(&mut self, token: Spanned>) { @@ -235,7 +236,7 @@ impl<'s> Parser<'s> { }; if let Some(color) = colored { - self.colorization.colors.push(Spanned { v: color, span: token.span }); + self.colorization.tokens.push(Spanned { v: color, span: token.span }); } } diff --git a/src/syntax/span.rs b/src/syntax/span.rs index e5c6912b8..546b3ad60 100644 --- a/src/syntax/span.rs +++ b/src/syntax/span.rs @@ -1,6 +1,6 @@ //! Spans map elements to the part of source code they originate from. -use std::fmt::{self, Display, Formatter}; +use std::fmt::{self, Debug, Display, Formatter}; /// Annotates a value with the part of the source code it corresponds to. @@ -28,13 +28,21 @@ impl Spanned { } } -impl Display for Spanned where T: std::fmt::Debug { +impl Display for Spanned where T: std::fmt::Display { fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!(f, "({:?}:{})", self.v, self.span) + write!(f, "({}, {}, ", self.span.start, self.span.end)?; + self.v.fmt(f)?; + write!(f, ")") } } -debug_display!(Spanned; T where T: std::fmt::Debug); +impl Debug for Spanned where T: std::fmt::Debug { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "({}, {}, ", self.span.start, self.span.end)?; + self.v.fmt(f)?; + write!(f, ")") + } +} /// Describes a slice of source code. #[derive(Copy, Clone, Eq, PartialEq, Hash)] @@ -68,7 +76,7 @@ impl Span { impl Display for Span { fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!(f, "[{}, {}]", self.start, self.end) + write!(f, "({}, {})", self.start, self.end) } } diff --git a/tests/layouts/coma.typ b/tests/layouter/coma.typ similarity index 100% rename from tests/layouts/coma.typ rename to tests/layouter/coma.typ diff --git a/tests/layouts/stack.typ b/tests/layouter/stack.typ similarity index 100% rename from tests/layouts/stack.typ rename to tests/layouter/stack.typ diff --git a/tests/parse.rs b/tests/parse.rs deleted file mode 100644 index 616f4d70b..000000000 --- a/tests/parse.rs +++ /dev/null @@ -1,236 +0,0 @@ -#![allow(unused_imports)] -#![allow(dead_code)] -#![allow(non_snake_case)] - -use typstc::func::Scope; -use typstc::size::Size; -use typstc::syntax::*; -use typstc::{function, parse}; - - -mod token_shorthands { - pub use super::Token::{ - Whitespace as W, - LineComment as LC, BlockComment as BC, StarSlash as SS, - LeftBracket as LB, RightBracket as RB, - LeftParen as LP, RightParen as RP, - LeftBrace as LBR, RightBrace as RBR, - Colon as CL, Comma as CM, Equals as EQ, - ExprIdent as ID, ExprStr as STR, ExprSize as SIZE, - ExprNumber as NUM, ExprBool as BOOL, - Star as ST, Underscore as U, Backtick as B, Text as T, - }; -} - -mod node_shorthands { - use super::Node; - pub use Node::{ - Space as S, Newline as N, Text, - ToggleItalic as I, ToggleBolder as B, ToggleMonospace as M, - Func, - }; - pub fn T(text: &str) -> Node { Node::Text(text.to_string()) } -} - -macro_rules! F { - (@body None) => (None); - (@body Some([$($tts:tt)*])) => ({ - let nodes = vec![$($tts)*].into_iter() - .map(|v| Spanned { v, span: Span::ZERO }) - .collect(); - - Some(SyntaxTree { nodes }) - }); - - ($($body:tt)*) => ({ - Func(FuncCall(Box::new(DebugFn { - pos: vec![], - key: vec![], - body: F!(@body $($body)*), - }))) - }); -} - -function! { - #[derive(Debug, PartialEq)] - pub struct DebugFn { - pos: Vec>, - key: Vec, - body: Option, - } - - parse(args, body, ctx) { - DebugFn { - pos: args.iter_pos().collect(), - key: args.iter_keys().collect(), - body: parse!(optional: body, ctx), - } - } - - layout() { vec![] } -} - -impl DebugFn { - fn compare(&self, other: &DebugFn) -> bool { - self.pos.iter().zip(&other.pos).all(|(a, b)| a.v == b.v) - && self.key.iter().zip(&other.key) - .all(|(a, b)| a.key.v == b.key.v && a.value.v == b.value.v) - && match (&self.body, &other.body) { - (Some(a), Some(b)) => compare(a, b), - (None, None) => true, - _ => false, - } - } -} - -fn downcast(func: &FuncCall) -> &DebugFn { - func.0.downcast::().expect("not a debug fn") -} - -fn compare(a: &SyntaxTree, b: &SyntaxTree) -> bool { - for (x, y) in a.nodes.iter().zip(&b.nodes) { - use node_shorthands::*; - let same = match (&x.v, &y.v) { - (S, S) | (N, N) | (I, I) | (B, B) | (M, M) => true, - (Text(t1), Text(t2)) => t1 == t2, - (Func(f1), Func(f2)) => { - downcast(f1).compare(downcast(f2)) - } - _ => false, - }; - - if !same { return false; } - } - true -} - -/// Parses the test syntax. -macro_rules! tokens { - ($($task:ident $src:expr =>($line:expr)=> [$($tts:tt)*])*) => ({ - #[allow(unused_mut)] - let mut cases = Vec::new(); - $(cases.push(($line, $src, tokens!(@$task [$($tts)*])));)* - cases - }); - - (@t [$($tts:tt)*]) => ({ - use token_shorthands::*; - Target::Tokenize(vec![$($tts)*]) - }); - - (@ts [$($tts:tt)*]) => ({ - use token_shorthands::*; - Target::TokenizeSpanned(tokens!(@__spans [$($tts)*])) - }); - - (@p [$($tts:tt)*]) => ({ - use node_shorthands::*; - - let nodes = vec![$($tts)*].into_iter() - .map(|v| Spanned { v, span: Span::ZERO }) - .collect(); - - Target::Parse(SyntaxTree { nodes }) - }); - - (@ps [$($tts:tt)*]) => ({ - use node_shorthands::*; - Target::ParseSpanned(tokens!(@__spans [$($tts)*])) - }); - - (@__spans [$(($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)),* $(,)?]) => ({ - vec![ - $(Spanned { v: $v, span: Span { - start: Position { line: $sl, column: $sc }, - end: Position { line: $el, column: $ec }, - }}),* - ] - }); -} - -#[derive(Debug)] -enum Target { - Tokenize(Vec>), - TokenizeSpanned(Vec>>), - Parse(SyntaxTree), - ParseSpanned(SyntaxTree), -} - -fn main() { - let tests = include!("cache/parse"); - let mut errors = false; - - let len = tests.len(); - println!(); - println!("Running {} test{}", len, if len > 1 { "s" } else { "" }); - - // Go through all test files. - for (file, cases) in tests.into_iter() { - print!("Testing: {}. ", file); - - let mut okay = 0; - let mut failed = 0; - - // Go through all tests in a test file. - for (line, src, target) in cases.into_iter() { - let (correct, expected, found) = test_case(src, target); - - // Check whether the tokenization works correctly. - if correct { - okay += 1; - } else { - if failed == 0 { - println!(); - } - - println!(" - Case failed in file {}.rs in line {}.", file, line); - println!(" - Source: {:?}", src); - println!(" - Expected: {:?}", expected); - println!(" - Found: {:?}", found); - println!(); - - failed += 1; - errors = true; - } - } - - // Print a small summary. - print!("{} okay, {} failed.", okay, failed); - if failed == 0 { - print!(" ✔") - } - println!(); - } - - println!(); - - if errors { - std::process::exit(-1); - } -} - -fn test_case(src: &str, target: Target) -> (bool, String, String) { - match target { - Target::Tokenize(tokens) => { - let found: Vec<_> = tokenize(src).map(Spanned::value).collect(); - (found == tokens, format!("{:?}", tokens), format!("{:?}", found)) - } - - Target::TokenizeSpanned(tokens) => { - let found: Vec<_> = tokenize(src).collect(); - (found == tokens, format!("{:?}", tokens), format!("{:?}", found)) - } - - Target::Parse(tree) => { - let scope = Scope::with_debug::(); - let (found, _, errs) = parse(src, ParseContext { scope: &scope }); - (compare(&tree, &found), format!("{:?}", tree), format!("{:?}", found)) - } - - Target::ParseSpanned(tree) => { - let scope = Scope::with_debug::(); - let (found, _, _) = parse(src, ParseContext { scope: &scope }); - (tree == found, format!("{:?}", tree), format!("{:?}", found)) - } - } -} diff --git a/tests/parsing/tokens.rs b/tests/parser/tokens.rs similarity index 92% rename from tests/parsing/tokens.rs rename to tests/parser/tokens.rs index 14f4e521e..fb48b32e9 100644 --- a/tests/parsing/tokens.rs +++ b/tests/parser/tokens.rs @@ -41,13 +41,13 @@ t "[a: true, x=1]" => [LB, ID("a"), CL, W(0), BOOL(true), CM, W(0), t "[120%]" => [LB, NUM(1.2), RB] // Body only tokens. -t "_*`" => [U, ST, B] -t "[func]*bold*" => [LB, ID("func"), RB, ST, T("bold"), ST] +t "_*`" => [U, S, B] +t "[func]*bold*" => [LB, ID("func"), RB, S, T("bold"), S] t "[_*`]" => [LB, T("_"), T("*"), T("`"), RB] t "hi_you_ there" => [T("hi"), U, T("you"), U, W(0), T("there")] // Nested functions. -t "[f: [=][*]]" => [LB, ID("f"), CL, W(0), LB, EQ, RB, LB, ST, RB, RB] +t "[f: [=][*]]" => [LB, ID("f"), CL, W(0), LB, EQ, RB, LB, S, RB, RB] t "[_][[,],]," => [LB, T("_"), RB, LB, LB, CM, RB, T(","), RB, T(",")] t "[=][=][=]" => [LB, EQ, RB, LB, T("="), RB, LB, EQ, RB] t "[=][[=][=][=]]" => [LB, EQ, RB, LB, LB, EQ, RB, LB, T("="), RB, LB, EQ, RB, RB] @@ -75,6 +75,6 @@ ts "[a=10]" => [(0:0, 0:1, LB), (0:1, 0:2, ID("a")), (0:2, 0:3, EQ), (0:3, 0:5, NUM(10.0)), (0:5, 0:6, RB)] ts r#"[x = "(1)"]*"# => [(0:0, 0:1, LB), (0:1, 0:2, ID("x")), (0:2, 0:3, W(0)), (0:3, 0:4, EQ), (0:4, 0:5, W(0)), (0:5, 0:10, STR("(1)")), - (0:10, 0:11, RB), (0:11, 0:12, ST)] + (0:10, 0:11, RB), (0:11, 0:12, S)] ts "// ab\r\n\nf" => [(0:0, 0:5, LC(" ab")), (0:5, 2:0, W(2)), (2:0, 2:1, T("f"))] ts "/*b*/_" => [(0:0, 0:5, BC("b")), (0:5, 0:6, U)] diff --git a/tests/parser/trees.rs b/tests/parser/trees.rs new file mode 100644 index 000000000..442f71dd7 --- /dev/null +++ b/tests/parser/trees.rs @@ -0,0 +1,33 @@ +p "" => [] +p "hi" => [T("hi")] +p "hi you" => [T("hi"), S, T("you")] +p "❤\n\n 🌍" => [T("❤"), N, T("🌍")] + +p "[func]" => [func!("func"; None)] +p "[tree][hi *you*]" => [func!("tree"; Some([T("hi"), S, B, T("you"), B]))] + +p "from [align: left] to" => [ + T("from"), S, func!("align", pos: [ID("left")]; None), S, T("to"), +] + +p "[box: x=1.2pt, false][a b c] bye" => [ + func!( + "box", + pos: [BOOL(false)], + key: ["x" => SIZE(Size::pt(1.2))]; + Some([T("a"), S, T("b"), S, T("c")]) + ), + S, T("bye"), +] + +c "hi" => [] +c "[align: left][\n _body_\n]" => [ + (0:0, 0:1, B), + (0:1, 0:6, FN), + (0:6, 0:7, CL), + (0:8, 0:12, ID), + (0:12, 0:13, B), + (0:13, 0:14, B), + (1:4, 1:10, IT), + (2:0, 2:2, B), +] diff --git a/tests/parsing/trees.rs b/tests/parsing/trees.rs deleted file mode 100644 index 78b168286..000000000 --- a/tests/parsing/trees.rs +++ /dev/null @@ -1,20 +0,0 @@ -p "" => [] -p "hi" => [T("hi")] -p "hi you" => [T("hi"), S, T("you")] -p "❤\n\n 🌍" => [T("❤"), N, T("🌍")] -p "[func]" => [F!(None)] -p "[tree][hi *you*]" => [F!(Some([T("hi"), S, B, T("you"), B]))] -// p "from [align: left] to" => [ -// T("from"), S, -// F!("align", pos=[ID("left")], None), -// S, T("to"), -// ] -// p "[box: x=1.2pt, false][a b c] bye" => [ -// F!( -// "box", -// pos=[BOOL(false)], -// key=["x": SIZE(Size::pt(1.2))], -// Some([T("a"), S, T("b"), S, T("c")]), -// ), -// S, T("bye"), -// ] diff --git a/tests/layout.rs b/tests/src/layouter.rs similarity index 96% rename from tests/layout.rs rename to tests/src/layouter.rs index 007b3c3f9..6d38666b0 100644 --- a/tests/layout.rs +++ b/tests/src/layouter.rs @@ -15,16 +15,17 @@ use typstc::style::PageStyle; use typstc::toddle::query::FileSystemFontProvider; use typstc::export::pdf::PdfExporter; -type Result = std::result::Result>; -fn main() -> Result<()> { +type DynResult = Result>; + +fn main() -> DynResult<()> { let opts = Options::parse(); create_dir_all("tests/cache/serial")?; create_dir_all("tests/cache/render")?; create_dir_all("tests/cache/pdf")?; - let tests: Vec<_> = read_dir("tests/layouts/")?.collect(); + let tests: Vec<_> = read_dir("tests/layouter/")?.collect(); let mut filtered = Vec::new(); for entry in tests { @@ -62,7 +63,7 @@ fn main() -> Result<()> { } /// Create a _PDF_ with a name from the source code. -fn test(name: &str, src: &str) -> Result<()> { +fn test(name: &str, src: &str) -> DynResult<()> { println!("Testing: {}.", name); let mut typesetter = Typesetter::new(); diff --git a/tests/src/parser.rs b/tests/src/parser.rs new file mode 100644 index 000000000..ecf1544c3 --- /dev/null +++ b/tests/src/parser.rs @@ -0,0 +1,311 @@ +use std::fmt::Debug; + +use typstc::func::Scope; +use typstc::size::Size; +use typstc::syntax::*; +use typstc::{function, parse}; + +mod spanless; +use spanless::SpanlessEq; + + +/// The result of a single test case. +enum Case { + Okay, + Failed { + line: usize, + src: &'static str, + expected: String, + found: String, + } +} + +/// Test all tests. +fn test(tests: Vec<(&str, Vec)>) { + println!(); + + let mut errors = false; + + let len = tests.len(); + println!("Running {} test{}", len, if len > 1 { "s" } else { "" }); + + for (file, cases) in tests { + print!("Testing: {}. ", file); + + let mut okay = 0; + let mut failed = 0; + + for case in cases { + match case { + Case::Okay => okay += 1, + Case::Failed { line, src, expected, found } => { + println!(); + println!(" - Case failed in file {}.rs in line {}.", file, line); + println!(" - Source: {:?}", src); + println!(" - Expected: {}", expected); + println!(" - Found: {}", found); + + failed += 1; + } + } + } + + // Print a small summary. + print!("{} okay, {} failed.", okay, failed); + if failed == 0 { + print!(" ✔") + } else { + errors = true; + } + + println!(); + } + + println!(); + + if errors { + std::process::exit(-1); + } +} + +/// The main test macro. +macro_rules! tokens { + ($($task:ident $src:expr =>($line:expr)=> [$($e:tt)*])*) => ({ + vec![$({ + let (okay, expected, found) = case!($task $src, [$($e)*]); + if okay { + Case::Okay + } else { + Case::Failed { + line: $line, + src: $src, + expected: format(expected), + found: format(found), + } + } + }),*] + }); +} + +//// Indented formatting for failed cases. +fn format(thing: impl Debug) -> String { + format!("{:#?}", thing).replace('\n', "\n ") +} + +/// Evaluates a single test. +macro_rules! case { + (t $($rest:tt)*) => (case!(@tokenize SpanlessEq::spanless_eq, $($rest)*)); + (ts $($rest:tt)*) => (case!(@tokenize PartialEq::eq, $($rest)*)); + + (@tokenize $cmp:expr, $src:expr, [$($e:tt)*]) => ({ + let expected = list!(tokens [$($e)*]); + let found = tokenize($src).collect::>(); + ($cmp(&found, &expected), expected, found) + }); + + (p $($rest:tt)*) => (case!(@parse SpanlessEq::spanless_eq, $($rest)*)); + (ps $($rest:tt)*) => (case!(@parse PartialEq::eq, $($rest)*)); + + (@parse $cmp:expr, $src:expr, [$($e:tt)*]) => ({ + let expected = SyntaxTree { nodes: list!(nodes [$($e)*]) }; + let found = parse($src, ParseContext { scope: &scope() }).0; + ($cmp(&found, &expected), expected, found) + }); + + (c $src:expr, [$($e:tt)*]) => ({ + let expected = Colorization { tokens: list!(colors [$($e)*]) }; + let found = parse($src, ParseContext { scope: &scope() }).1; + (expected == found, expected, found) + }); + + (e $src:expr, [$($e:tt)*]) => ({ + let expected = ErrorMap { errors: list!([$($e)*]) }; + let found = parse($src, ParseContext { scope: &scope() }).2; + (expected == found, expected, found) + }); +} + +/// A scope containing the `DebugFn` as a fallback. +fn scope() -> Scope { + Scope::with_debug::() +} + +/// Parses possibly-spanned lists of token or node expressions. +macro_rules! list { + (expr [$($item:expr),* $(,)?]) => ({ + #[allow(unused_imports)] + use cuts::expr::*; + Tuple { items: vec![$(zspan($item)),*] } + }); + + (expr [$($key:expr =>($_:expr)=> $value:expr),* $(,)?]) => ({ + #[allow(unused_imports)] + use cuts::expr::*; + Object { + pairs: vec![$(Pair { + key: zspan(Ident($key.to_string())), + value: zspan($value), + }),*] + } + }); + + ($cut:ident [$($e:tt)*]) => ({ + #[allow(unused_imports)] + use cuts::$cut::*; + list!([$($e)*]) + }); + + ([$(($sl:tt:$sc:tt, $el:tt:$ec:tt, $v:expr)),* $(,)?]) => ({ + vec![ + $(Spanned { v: $v, span: Span { + start: Position { line: $sl, column: $sc }, + end: Position { line: $el, column: $ec }, + }}),* + ] + }); + + ([$($e:tt)*]) => (vec![$($e)*].into_iter().map(zspan).collect::>()); +} + +/// Composes a function expression. +macro_rules! func { + ($name:expr $(,pos: [$($p:tt)*])? $(,key: [$($k:tt)*])?; $($b:tt)*) => ({ + #![allow(unused_mut, unused_assignments)] + + let mut positional = Tuple::new(); + let mut keyword = Object::new(); + + $(positional = list!(expr [$($p)*]);)? + $(keyword = list!(expr [$($k)*]);)? + + Node::Func(FuncCall(Box::new(DebugFn { + header: FuncHeader { + name: zspan(Ident($name.to_string())), + args: FuncArgs { + positional, + keyword, + }, + }, + body: func!(@body $($b)*), + }))) + }); + + (@body Some($($b:tt)*)) => (Some(SyntaxTree { nodes: list!(nodes $($b)*) })); + (@body None) => (None); +} + +function! { + /// Most functions in the tests are parsed into the debug function for easy + /// inspection of arguments and body. + #[derive(Debug, PartialEq)] + pub struct DebugFn { + header: FuncHeader, + body: Option, + } + + parse(header, body, ctx) { + DebugFn { + header: header.clone(), + body: parse!(optional: body, ctx), + } + } + + layout() { vec![] } +} + +/// Span an element with a zero span. +fn zspan(v: T) -> Spanned { + Spanned { v, span: Span::ZERO } +} + +/// Abbreviations for tokens, nodes, colors and expressions. +#[allow(non_snake_case, dead_code)] +mod cuts { + pub mod tokens { + pub use typstc::syntax::Token::{ + Whitespace as W, + LineComment as LC, + BlockComment as BC, + StarSlash as SS, + LeftBracket as LB, + RightBracket as RB, + LeftParen as LP, + RightParen as RP, + LeftBrace as LBR, + RightBrace as RBR, + Colon as CL, + Comma as CM, + Equals as EQ, + ExprIdent as ID, + ExprStr as STR, + ExprSize as SIZE, + ExprNumber as NUM, + ExprBool as BOOL, + Star as S, + Underscore as U, + Backtick as B, + Text as T, + }; + } + + pub mod nodes { + use typstc::syntax::Node; + + pub use Node::{ + Space as S, + Newline as N, + ToggleItalic as I, + ToggleBolder as B, + ToggleMonospace as M, + }; + + pub fn T(text: &str) -> Node { + Node::Text(text.to_string()) + } + } + + pub mod colors { + pub use typstc::syntax::ColorToken::{ + Comment as C, + Bracket as B, + FuncName as FN, + Colon as CL, + Key as K, + Equals as EQ, + Comma as CM, + Paren as P, + Brace as BR, + ExprIdent as ID, + ExprStr as STR, + ExprNumber as NUM, + ExprSize as SIZE, + ExprBool as BOOL, + Bold as BD, + Italic as IT, + Monospace as MS, + Invalid as INV, + }; + } + + pub mod expr { + use typstc::syntax::{Expression, Ident}; + + pub use Expression::{ + Number as NUM, + Size as SIZE, + Bool as BOOL, + }; + + pub fn ID(text: &str) -> Expression { + Expression::Ident(Ident(text.to_string())) + } + + pub fn STR(text: &str) -> Expression { + Expression::Str(text.to_string()) + } + } +} + +fn main() { + test(include!("../cache/parser-tests.rs")) +} diff --git a/tests/render.py b/tests/src/render.py similarity index 93% rename from tests/render.py rename to tests/src/render.py index 1387ed534..bb27e9732 100644 --- a/tests/render.py +++ b/tests/src/render.py @@ -7,7 +7,7 @@ from PIL import Image, ImageDraw, ImageFont BASE = os.path.dirname(__file__) -CACHE = os.path.join(BASE, 'cache/') +CACHE = os.path.join(BASE, '../cache/') SERIAL = os.path.join(CACHE, 'serial/') RENDER = os.path.join(CACHE, 'render/') @@ -98,16 +98,18 @@ class MultiboxRenderer: class BoxRenderer: - def __init__(self, fonts, width, height): + def __init__(self, fonts, width, height, grid=False): self.fonts = fonts self.size = (pix(width), pix(height)) img = Image.new('RGBA', self.size, (255, 255, 255, 255)) pixels = numpy.array(img) - # for i in range(0, int(height)): - # for j in range(0, int(width)): - # if ((i // 2) % 2 == 0) == ((j // 2) % 2 == 0): - # pixels[4*i:4*(i+1), 4*j:4*(j+1)] = (225, 225, 225, 255) + + if grid: + for i in range(0, int(height)): + for j in range(0, int(width)): + if ((i // 2) % 2 == 0) == ((j // 2) % 2 == 0): + pixels[4*i:4*(i+1), 4*j:4*(j+1)] = (225, 225, 225, 255) self.img = Image.fromarray(pixels, 'RGBA') self.draw = ImageDraw.Draw(self.img) diff --git a/tests/src/spanless.rs b/tests/src/spanless.rs new file mode 100644 index 000000000..fde5a2edb --- /dev/null +++ b/tests/src/spanless.rs @@ -0,0 +1,62 @@ +use super::*; + + +/// Compares elements by only looking at values and ignoring spans. +pub trait SpanlessEq { + fn spanless_eq(&self, other: &T) -> bool; +} + +impl SpanlessEq>>> for Vec>> { + fn spanless_eq(&self, other: &Vec>) -> bool { + self.len() == other.len() + && self.iter().zip(other).all(|(x, y)| x.v == y.v) + } +} + +impl SpanlessEq for SyntaxTree { + fn spanless_eq(&self, other: &SyntaxTree) -> bool { + fn downcast(func: &FuncCall) -> &DebugFn { + func.0.downcast::().expect("not a debug fn") + } + + self.nodes.len() == other.nodes.len() + && self.nodes.iter().zip(&other.nodes).all(|(x, y)| match (&x.v, &y.v) { + (Node::Func(a), Node::Func(b)) => downcast(a).spanless_eq(downcast(b)), + (a, b) => a == b, + }) + } +} + +impl SpanlessEq for DebugFn { + fn spanless_eq(&self, other: &DebugFn) -> bool { + self.header.name.v == other.header.name.v + && self.header.args.positional.spanless_eq(&other.header.args.positional) + && self.header.args.keyword.spanless_eq(&other.header.args.keyword) + } +} + +impl SpanlessEq for Expression { + fn spanless_eq(&self, other: &Expression) -> bool { + match (self, other) { + (Expression::Tuple(a), Expression::Tuple(b)) => a.spanless_eq(b), + (Expression::Object(a), Expression::Object(b)) => a.spanless_eq(b), + (a, b) => a == b, + } + } +} + +impl SpanlessEq for Tuple { + fn spanless_eq(&self, other: &Tuple) -> bool { + self.items.len() == other.items.len() + && self.items.iter().zip(&other.items) + .all(|(x, y)| x.v.spanless_eq(&y.v)) + } +} + +impl SpanlessEq for Object { + fn spanless_eq(&self, other: &Object) -> bool { + self.pairs.len() == other.pairs.len() + && self.pairs.iter().zip(&other.pairs) + .all(|(x, y)| x.key.v == y.key.v && x.value.v.spanless_eq(&y.value.v)) + } +}