From 59d811aeba4491d54d2b0220109fd21a8f838b9b Mon Sep 17 00:00:00 2001 From: Laurenz Date: Wed, 6 Jan 2021 21:06:48 +0100 Subject: [PATCH] =?UTF-8?q?Inline=20literal=20enum=20into=20expression=20e?= =?UTF-8?q?num=20=F0=9F=94=80?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/eval/mod.rs | 82 ++++++--------- src/eval/value.rs | 7 +- src/geom/relative.rs | 2 +- src/parse/collection.rs | 4 +- src/parse/mod.rs | 36 +++---- src/parse/tests.rs | 37 +------ src/parse/tokens.rs | 226 ++++++++++++++++++++-------------------- src/pretty.rs | 2 +- src/syntax/expr.rs | 75 ++++++------- src/syntax/node.rs | 4 - src/syntax/token.rs | 50 ++++----- 11 files changed, 231 insertions(+), 294 deletions(-) diff --git a/src/eval/mod.rs b/src/eval/mod.rs index 1c6e3d51f..68a97b43d 100644 --- a/src/eval/mod.rs +++ b/src/eval/mod.rs @@ -75,7 +75,6 @@ impl Eval for Spanned<&Node> { let node = ctx.make_text_node(text.clone()); ctx.push(node); } - Node::Space => { let em = ctx.state.font.font_size(); ctx.push(NodeSpacing { @@ -85,13 +84,10 @@ impl Eval for Spanned<&Node> { } Node::Linebreak => ctx.apply_linebreak(), Node::Parbreak => ctx.apply_parbreak(), - Node::Strong => ctx.state.font.strong ^= true, Node::Emph => ctx.state.font.emph ^= true, - Node::Heading(heading) => heading.with_span(self.span).eval(ctx), Node::Raw(raw) => raw.with_span(self.span).eval(ctx), - Node::Expr(expr) => { let value = expr.with_span(self.span).eval(ctx); value.eval(ctx) @@ -153,7 +149,21 @@ impl Eval for Spanned<&Expr> { fn eval(self, ctx: &mut EvalContext) -> Self::Output { match self.v { - Expr::Lit(v) => v.with_span(self.span).eval(ctx), + Expr::None => Value::None, + Expr::Ident(v) => match ctx.state.scope.get(v) { + Some(value) => value.clone(), + None => { + ctx.diag(error!(self.span, "unknown variable")); + Value::Error + } + }, + Expr::Bool(v) => Value::Bool(*v), + Expr::Int(v) => Value::Int(*v), + Expr::Float(v) => Value::Float(*v), + Expr::Length(v, unit) => Value::Length(Length::with_unit(*v, *unit)), + Expr::Percent(v) => Value::Relative(Relative::new(v / 100.0)), + Expr::Color(v) => Value::Color(Color::Rgba(*v)), + Expr::Str(v) => Value::Str(v.clone()), Expr::Call(v) => v.with_span(self.span).eval(ctx), Expr::Unary(v) => v.with_span(self.span).eval(ctx), Expr::Binary(v) => v.with_span(self.span).eval(ctx), @@ -164,49 +174,6 @@ impl Eval for Spanned<&Expr> { } } -impl Eval for Spanned<&Lit> { - type Output = Value; - - fn eval(self, ctx: &mut EvalContext) -> Self::Output { - match *self.v { - Lit::Ident(ref v) => match ctx.state.scope.get(&v) { - Some(value) => value.clone(), - None => { - ctx.diag(error!(self.span, "unknown variable")); - Value::Error - } - }, - Lit::None => Value::None, - Lit::Bool(v) => Value::Bool(v), - Lit::Int(v) => Value::Int(v), - Lit::Float(v) => Value::Float(v), - Lit::Length(v, unit) => Value::Length(Length::with_unit(v, unit)), - Lit::Percent(v) => Value::Relative(Relative::new(v / 100.0)), - Lit::Color(v) => Value::Color(Color::Rgba(v)), - Lit::Str(ref v) => Value::Str(v.clone()), - } - } -} - -impl Eval for Spanned<&ExprArray> { - type Output = ValueArray; - - fn eval(self, ctx: &mut EvalContext) -> Self::Output { - self.v.iter().map(|expr| expr.as_ref().eval(ctx)).collect() - } -} - -impl Eval for Spanned<&ExprDict> { - type Output = ValueDict; - - fn eval(self, ctx: &mut EvalContext) -> Self::Output { - self.v - .iter() - .map(|Named { name, expr }| (name.v.0.clone(), expr.as_ref().eval(ctx))) - .collect() - } -} - impl Eval for Spanned<&ExprUnary> { type Output = Value; @@ -245,6 +212,25 @@ impl Eval for Spanned<&ExprBinary> { } } +impl Eval for Spanned<&ExprArray> { + type Output = ValueArray; + + fn eval(self, ctx: &mut EvalContext) -> Self::Output { + self.v.iter().map(|expr| expr.as_ref().eval(ctx)).collect() + } +} + +impl Eval for Spanned<&ExprDict> { + type Output = ValueDict; + + fn eval(self, ctx: &mut EvalContext) -> Self::Output { + self.v + .iter() + .map(|Named { name, expr }| (name.v.0.clone(), expr.as_ref().eval(ctx))) + .collect() + } +} + /// Compute the negation of a value. fn neg(ctx: &mut EvalContext, span: Span, value: Value) -> Value { use Value::*; diff --git a/src/eval/value.rs b/src/eval/value.rs index 507419879..708f4f044 100644 --- a/src/eval/value.rs +++ b/src/eval/value.rs @@ -487,6 +487,7 @@ macro_rules! impl_type { mod tests { use super::*; use crate::color::RgbaColor; + use crate::parse::parse; use crate::pretty::pretty; use crate::syntax::Node; @@ -496,7 +497,7 @@ mod tests { } #[test] - fn test_pretty_print_values() { + fn test_pretty_print_simple_values() { test_pretty(Value::None, "none"); test_pretty(false, "false"); test_pretty(12.4, "12.4"); @@ -521,8 +522,8 @@ mod tests { // Dictionary. let mut dict = BTreeMap::new(); dict.insert("one".into(), Value::Int(1)); - dict.insert("two".into(), Value::Int(2)); + dict.insert("two".into(), Value::Content(parse("[f]").output)); test_pretty(BTreeMap::new(), "(:)"); - test_pretty(dict, "(one: 1, two: 2)"); + test_pretty(dict, "(one: 1, two: [f])"); } } diff --git a/src/geom/relative.rs b/src/geom/relative.rs index fdd1f9bb6..8fd430af8 100644 --- a/src/geom/relative.rs +++ b/src/geom/relative.rs @@ -3,7 +3,7 @@ use super::*; /// A relative length. /// /// _Note_: `50%` is represented as `0.5` here, but stored as `50.0` in the -/// corresponding [literal](crate::syntax::Lit::Percent). +/// corresponding [literal](crate::syntax::Expr::Percent). #[derive(Default, Copy, Clone, PartialEq, PartialOrd)] pub struct Relative(f64); diff --git a/src/parse/collection.rs b/src/parse/collection.rs index 889cfb0fb..98d4219f7 100644 --- a/src/parse/collection.rs +++ b/src/parse/collection.rs @@ -53,7 +53,7 @@ fn collection(p: &mut Parser, mut collection: T) -> T { fn argument(p: &mut Parser) -> Option { let first = p.span_if(expr)?; if p.eat_if(Token::Colon) { - if let Expr::Lit(Lit::Ident(ident)) = first.v { + if let Expr::Ident(ident) = first.v { let expr = p.span_if(expr)?; let name = ident.with_span(first.span); p.deco(Deco::Name.with_span(name.span)); @@ -131,7 +131,7 @@ impl Collection for State { fn take(expr: &mut Spanned) -> Spanned { // Replace with anything, it's overwritten anyway. - std::mem::replace(expr, Spanned::zero(Expr::Lit(Lit::Bool(false)))) + std::mem::replace(expr, Spanned::zero(Expr::Bool(false))) } fn diag(p: &mut Parser, arg: Spanned) { diff --git a/src/parse/mod.rs b/src/parse/mod.rs index ef4ce46fc..150b5ed17 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -49,6 +49,7 @@ fn tree(p: &mut Parser) -> Tree { /// Parse a syntax node. fn node(p: &mut Parser, at_start: bool) -> Option { let node = match p.peek()? { + Token::Text(text) => Node::Text(text.into()), Token::Space(newlines) => { if newlines < 2 { Node::Space @@ -56,13 +57,20 @@ fn node(p: &mut Parser, at_start: bool) -> Option { Node::Parbreak } } - Token::Text(text) => Node::Text(text.into()), Token::LineComment(_) | Token::BlockComment(_) => { p.eat(); return None; } + Token::LeftBracket => { + return Some(Node::Expr(Expr::Call(bracket_call(p)))); + } + + Token::LeftBrace => { + return Some(Node::Expr(block_expr(p)?)); + } + Token::Star => Node::Strong, Token::Underscore => Node::Emph, Token::Tilde => Node::Text("\u{00A0}".into()), @@ -77,14 +85,6 @@ fn node(p: &mut Parser, at_start: bool) -> Option { Token::Raw(t) => Node::Raw(raw(p, t)), Token::UnicodeEscape(t) => Node::Text(unicode_escape(p, t)), - Token::LeftBracket => { - return Some(Node::Expr(Expr::Call(bracket_call(p)))); - } - - Token::LeftBrace => { - return Some(Node::Expr(block_expr(p)?)); - } - _ => { p.diag_unexpected(); return None; @@ -321,19 +321,19 @@ fn value(p: &mut Parser) -> Option { let name = ident.with_span(p.peek_span()); return Some(Expr::Call(paren_call(p, name))); } else { - return Some(Expr::Lit(Lit::Ident(ident))); + return Some(Expr::Ident(ident)); } } // Basic values. - Some(Token::None) => Expr::Lit(Lit::None), - Some(Token::Bool(b)) => Expr::Lit(Lit::Bool(b)), - Some(Token::Int(i)) => Expr::Lit(Lit::Int(i)), - Some(Token::Float(f)) => Expr::Lit(Lit::Float(f)), - Some(Token::Length(val, unit)) => Expr::Lit(Lit::Length(val, unit)), - Some(Token::Percent(p)) => Expr::Lit(Lit::Percent(p)), - Some(Token::Hex(hex)) => Expr::Lit(Lit::Color(color(p, hex))), - Some(Token::Str(token)) => Expr::Lit(Lit::Str(str(p, token))), + Some(Token::None) => Expr::None, + Some(Token::Bool(b)) => Expr::Bool(b), + Some(Token::Int(i)) => Expr::Int(i), + Some(Token::Float(f)) => Expr::Float(f), + Some(Token::Length(val, unit)) => Expr::Length(val, unit), + Some(Token::Percent(p)) => Expr::Percent(p), + Some(Token::Hex(hex)) => Expr::Color(color(p, hex)), + Some(Token::Str(token)) => Expr::Str(str(p, token)), // No value. _ => { diff --git a/src/parse/tests.rs b/src/parse/tests.rs index 8de03aff5..701d2a730 100644 --- a/src/parse/tests.rs +++ b/src/parse/tests.rs @@ -9,7 +9,8 @@ use crate::geom::Unit; use crate::syntax::*; use BinOp::*; -use Node::{Emph, Linebreak, Parbreak, Space, Strong}; +use Expr::{Bool, Color, Float, Int, Length, Percent}; +use Node::{Emph, Expr as Block, Linebreak, Parbreak, Space, Strong}; use UnOp::*; macro_rules! t { @@ -99,39 +100,11 @@ fn Raw(lang: Option<&str>, lines: &[&str], inline: bool) -> Node { } fn Id(ident: &str) -> Expr { - Expr::Lit(Lit::Ident(Ident(ident.to_string()))) -} - -fn Bool(b: bool) -> Expr { - Expr::Lit(Lit::Bool(b)) -} - -fn Int(int: i64) -> Expr { - Expr::Lit(Lit::Int(int)) -} - -fn Float(float: f64) -> Expr { - Expr::Lit(Lit::Float(float)) -} - -fn Percent(percent: f64) -> Expr { - Expr::Lit(Lit::Percent(percent)) -} - -fn Length(val: f64, unit: Unit) -> Expr { - Expr::Lit(Lit::Length(val, unit)) -} - -fn Color(color: RgbaColor) -> Expr { - Expr::Lit(Lit::Color(color)) + Expr::Ident(Ident(ident.to_string())) } fn Str(string: &str) -> Expr { - Expr::Lit(Lit::Str(string.to_string())) -} - -fn Block(expr: Expr) -> Node { - Node::Expr(expr) + Expr::Str(string.to_string()) } fn Binary( @@ -614,7 +587,7 @@ fn test_parse_values() { t!("{name}" Block(Id("name"))); t!("{ke-bab}" Block(Id("ke-bab"))); t!("{Ξ±}" Block(Id("Ξ±"))); - t!("{none}" Block(Expr::Lit(Lit::None))); + t!("{none}" Block(Expr::None)); t!("{true}" Block(Bool(true))); t!("{false}" Block(Bool(false))); t!("{1.0e-4}" Block(Float(1e-4))); diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs index 74ec47e9f..dee921684 100644 --- a/src/parse/tokens.rs +++ b/src/parse/tokens.rs @@ -379,25 +379,21 @@ mod tests { use crate::parse::tests::check; use Option::None; - use Token::{ - BlockComment as BC, Ident as Id, LeftBrace as LB, LeftBracket as L, - LeftParen as LP, LineComment as LC, RightBrace as RB, RightBracket as R, - RightParen as RP, Space as S, Text as T, *, - }; + use Token::{Ident, *}; use Unit::*; - fn Str(string: &str, terminated: bool) -> Token { - Token::Str(TokenStr { string, terminated }) - } - fn Raw(text: &str, backticks: usize, terminated: bool) -> Token { Token::Raw(TokenRaw { text, backticks, terminated }) } - fn UE(sequence: &str, terminated: bool) -> Token { + fn UnicodeEscape(sequence: &str, terminated: bool) -> Token { Token::UnicodeEscape(TokenUnicodeEscape { sequence, terminated }) } + fn Str(string: &str, terminated: bool) -> Token { + Token::Str(TokenStr { string, terminated }) + } + /// Building blocks for suffix testing. /// /// We extend each test case with a collection of different suffixes to make @@ -421,27 +417,27 @@ mod tests { /// - the resulting suffix token const SUFFIXES: &[(char, Option, &str, Token)] = &[ // Whitespace suffixes. - (' ', None, " ", S(0)), - (' ', None, "\n", S(1)), - (' ', None, "\r", S(1)), - (' ', None, "\r\n", S(1)), + (' ', None, " ", Space(0)), + (' ', None, "\n", Space(1)), + (' ', None, "\r", Space(1)), + (' ', None, "\r\n", Space(1)), // Letter suffixes. - ('a', Some(Body), "hello", T("hello")), - ('a', Some(Body), "πŸ’š", T("πŸ’š")), - ('a', Some(Header), "val", Id("val")), - ('a', Some(Header), "Ξ±", Id("Ξ±")), - ('a', Some(Header), "_", Id("_")), + ('a', Some(Body), "hello", Text("hello")), + ('a', Some(Body), "πŸ’š", Text("πŸ’š")), + ('a', Some(Header), "val", Ident("val")), + ('a', Some(Header), "Ξ±", Ident("Ξ±")), + ('a', Some(Header), "_", Ident("_")), // Number suffixes. ('1', Some(Header), "2", Int(2)), ('1', Some(Header), ".2", Float(0.2)), // Symbol suffixes. - ('/', None, "[", L), - ('/', None, "//", LC("")), - ('/', None, "/**/", BC("")), + ('/', None, "[", LeftBracket), + ('/', None, "//", LineComment("")), + ('/', None, "/**/", BlockComment("")), ('/', Some(Body), "*", Star), ('/', Some(Body), "_", Underscore), - ('/', Some(Body), r"\\", T(r"\")), - ('/', Some(Header), "(", LP), + ('/', Some(Body), r"\\", Text(r"\")), + ('/', Some(Header), "(", LeftParen), ('/', Some(Header), ":", Colon), ('/', Some(Header), "+", Plus), ('/', Some(Header), "#123", Hex("123")), @@ -487,69 +483,69 @@ mod tests { fn test_tokenize_whitespace() { // Test basic whitespace. t!(Both["a1/"]: "" => ); - t!(Both["a1/"]: " " => S(0)); - t!(Both["a1/"]: " " => S(0)); - t!(Both["a1/"]: "\t" => S(0)); - t!(Both["a1/"]: " \t" => S(0)); - t!(Both["a1/"]: "\u{202F}" => S(0)); + t!(Both["a1/"]: " " => Space(0)); + t!(Both["a1/"]: " " => Space(0)); + t!(Both["a1/"]: "\t" => Space(0)); + t!(Both["a1/"]: " \t" => Space(0)); + t!(Both["a1/"]: "\u{202F}" => Space(0)); // Test newline counting. - t!(Both["a1/"]: "\n" => S(1)); - t!(Both["a1/"]: "\n " => S(1)); - t!(Both["a1/"]: " \n" => S(1)); - t!(Both["a1/"]: " \n " => S(1)); - t!(Both["a1/"]: "\r\n" => S(1)); - t!(Both["a1/"]: " \n\t \n " => S(2)); - t!(Both["a1/"]: "\n\r" => S(2)); - t!(Both["a1/"]: " \r\r\n \x0D" => S(3)); + t!(Both["a1/"]: "\n" => Space(1)); + t!(Both["a1/"]: "\n " => Space(1)); + t!(Both["a1/"]: " \n" => Space(1)); + t!(Both["a1/"]: " \n " => Space(1)); + t!(Both["a1/"]: "\r\n" => Space(1)); + t!(Both["a1/"]: " \n\t \n " => Space(2)); + t!(Both["a1/"]: "\n\r" => Space(2)); + t!(Both["a1/"]: " \r\r\n \x0D" => Space(3)); } #[test] fn test_tokenize_line_comments() { // Test line comment with no trailing newline. - t!(Both[""]: "//" => LC("")); + t!(Both[""]: "//" => LineComment("")); // Test line comment ends at newline. - t!(Both["a1/"]: "//bc\n" => LC("bc"), S(1)); - t!(Both["a1/"]: "// bc \n" => LC(" bc "), S(1)); - t!(Both["a1/"]: "//bc\r\n" => LC("bc"), S(1)); + t!(Both["a1/"]: "//bc\n" => LineComment("bc"), Space(1)); + t!(Both["a1/"]: "// bc \n" => LineComment(" bc "), Space(1)); + t!(Both["a1/"]: "//bc\r\n" => LineComment("bc"), Space(1)); // Test nested line comments. - t!(Both["a1/"]: "//a//b\n" => LC("a//b"), S(1)); + t!(Both["a1/"]: "//a//b\n" => LineComment("a//b"), Space(1)); } #[test] fn test_tokenize_block_comments() { // Test basic block comments. - t!(Both[""]: "/*" => BC("")); - t!(Both: "/**/" => BC("")); - t!(Both: "/*🏞*/" => BC("🏞")); - t!(Both: "/*\n*/" => BC("\n")); + t!(Both[""]: "/*" => BlockComment("")); + t!(Both: "/**/" => BlockComment("")); + t!(Both: "/*🏞*/" => BlockComment("🏞")); + t!(Both: "/*\n*/" => BlockComment("\n")); // Test depth 1 and 2 nested block comments. - t!(Both: "/* /* */ */" => BC(" /* */ ")); - t!(Both: "/*/*/**/*/*/" => BC("/*/**/*/")); + t!(Both: "/* /* */ */" => BlockComment(" /* */ ")); + t!(Both: "/*/*/**/*/*/" => BlockComment("/*/**/*/")); // Test two nested, one unclosed block comments. - t!(Both[""]: "/*/*/**/*/" => BC("/*/**/*/")); + t!(Both[""]: "/*/*/**/*/" => BlockComment("/*/**/*/")); // Test all combinations of up to two following slashes and stars. - t!(Both[""]: "/*" => BC("")); - t!(Both[""]: "/*/" => BC("/")); - t!(Both[""]: "/**" => BC("*")); - t!(Both[""]: "/*//" => BC("//")); - t!(Both[""]: "/*/*" => BC("/*")); - t!(Both[""]: "/**/" => BC("")); - t!(Both[""]: "/***" => BC("**")); + t!(Both[""]: "/*" => BlockComment("")); + t!(Both[""]: "/*/" => BlockComment("/")); + t!(Both[""]: "/**" => BlockComment("*")); + t!(Both[""]: "/*//" => BlockComment("//")); + t!(Both[""]: "/*/*" => BlockComment("/*")); + t!(Both[""]: "/**/" => BlockComment("")); + t!(Both[""]: "/***" => BlockComment("**")); } #[test] fn test_tokenize_body_tokens() { // Test parentheses. - t!(Body: "[" => L); - t!(Body: "]" => R); - t!(Body: "{" => LB); - t!(Body: "}" => RB); + t!(Body: "[" => LeftBracket); + t!(Body: "]" => RightBracket); + t!(Body: "{" => LeftBrace); + t!(Body: "}" => RightBrace); // Test markup tokens. t!(Body[" a1"]: "*" => Star); @@ -559,7 +555,7 @@ mod tests { t!(Body[" "]: r"\" => Backslash); // Test header symbols. - t!(Body[" /"]: ":,=|/+-" => T(":,=|/+-")); + t!(Body[" /"]: ":,=|/+-" => Text(":,=|/+-")); } #[test] @@ -584,62 +580,62 @@ mod tests { #[test] fn test_tokenize_escape_sequences() { // Test escapable symbols. - t!(Body: r"\\" => T(r"\")); - t!(Body: r"\/" => T("/")); - t!(Body: r"\[" => T("[")); - t!(Body: r"\]" => T("]")); - t!(Body: r"\{" => T("{")); - t!(Body: r"\}" => T("}")); - t!(Body: r"\*" => T("*")); - t!(Body: r"\_" => T("_")); - t!(Body: r"\#" => T("#")); - t!(Body: r"\~" => T("~")); - t!(Body: r"\`" => T("`")); + t!(Body: r"\\" => Text(r"\")); + t!(Body: r"\/" => Text("/")); + t!(Body: r"\[" => Text("[")); + t!(Body: r"\]" => Text("]")); + t!(Body: r"\{" => Text("{")); + t!(Body: r"\}" => Text("}")); + t!(Body: r"\*" => Text("*")); + t!(Body: r"\_" => Text("_")); + t!(Body: r"\#" => Text("#")); + t!(Body: r"\~" => Text("~")); + t!(Body: r"\`" => Text("`")); // Test unescapable symbols. - t!(Body[" /"]: r"\a" => T(r"\"), T("a")); - t!(Body[" /"]: r"\u" => T(r"\"), T("u")); - t!(Body[" /"]: r"\1" => T(r"\"), T("1")); - t!(Body[" /"]: r"\:" => T(r"\"), T(":")); - t!(Body[" /"]: r"\=" => T(r"\"), T("=")); - t!(Body[" /"]: r#"\""# => T(r"\"), T("\"")); + t!(Body[" /"]: r"\a" => Text(r"\"), Text("a")); + t!(Body[" /"]: r"\u" => Text(r"\"), Text("u")); + t!(Body[" /"]: r"\1" => Text(r"\"), Text("1")); + t!(Body[" /"]: r"\:" => Text(r"\"), Text(":")); + t!(Body[" /"]: r"\=" => Text(r"\"), Text("=")); + t!(Body[" /"]: r#"\""# => Text(r"\"), Text("\"")); // Test basic unicode escapes. - t!(Body: r"\u{}" => UE("", true)); - t!(Body: r"\u{2603}" => UE("2603", true)); - t!(Body: r"\u{P}" => UE("P", true)); + t!(Body: r"\u{}" => UnicodeEscape("", true)); + t!(Body: r"\u{2603}" => UnicodeEscape("2603", true)); + t!(Body: r"\u{P}" => UnicodeEscape("P", true)); // Test unclosed unicode escapes. - t!(Body[" /"]: r"\u{" => UE("", false)); - t!(Body[" /"]: r"\u{1" => UE("1", false)); - t!(Body[" /"]: r"\u{26A4" => UE("26A4", false)); - t!(Body[" /"]: r"\u{1Q3P" => UE("1Q3P", false)); - t!(Body: r"\u{1πŸ•}" => UE("1", false), T("πŸ•"), RB); + t!(Body[" /"]: r"\u{" => UnicodeEscape("", false)); + t!(Body[" /"]: r"\u{1" => UnicodeEscape("1", false)); + t!(Body[" /"]: r"\u{26A4" => UnicodeEscape("26A4", false)); + t!(Body[" /"]: r"\u{1Q3P" => UnicodeEscape("1Q3P", false)); + t!(Body: r"\u{1πŸ•}" => UnicodeEscape("1", false), Text("πŸ•"), RightBrace); } #[test] fn test_tokenize_text() { // Test basic text. - t!(Body[" /"]: "hello" => T("hello")); - t!(Body[" /"]: "hello-world" => T("hello-world")); + t!(Body[" /"]: "hello" => Text("hello")); + t!(Body[" /"]: "hello-world" => Text("hello-world")); // Test header symbols in text. - t!(Body[" /"]: "a():\"b" => T("a():\"b")); + t!(Body[" /"]: "a():\"b" => Text("a():\"b")); // Test text ends. - t!(Body[""]: "hello " => T("hello"), S(0)); - t!(Body[""]: "hello~" => T("hello"), Tilde); + t!(Body[""]: "hello " => Text("hello"), Space(0)); + t!(Body[""]: "hello~" => Text("hello"), Tilde); } #[test] fn test_tokenize_header_tokens() { // Test parentheses. - t!(Header: "[" => L); - t!(Header: "]" => R); - t!(Header: "{" => LB); - t!(Header: "}" => RB); - t!(Header: "(" => LP); - t!(Header: ")" => RP); + t!(Header: "[" => LeftBracket); + t!(Header: "]" => RightBracket); + t!(Header: "{" => LeftBrace); + t!(Header: "}" => RightBrace); + t!(Header: "(" => LeftParen); + t!(Header: ")" => RightParen); // Test structural tokens. t!(Header: ":" => Colon); @@ -652,10 +648,10 @@ mod tests { // Test hyphen parsed as symbol. t!(Header[" /"]: "-1" => Hyphen, Int(1)); - t!(Header[" /"]: "-a" => Hyphen, Id("a")); + t!(Header[" /"]: "-a" => Hyphen, Ident("a")); t!(Header[" /"]: "--1" => Hyphen, Hyphen, Int(1)); - t!(Header[" /"]: "--_a" => Hyphen, Hyphen, Id("_a")); - t!(Header[" /"]: "a-b" => Id("a-b")); + t!(Header[" /"]: "--_a" => Hyphen, Hyphen, Ident("_a")); + t!(Header[" /"]: "a-b" => Ident("a-b")); // Test some operations. t!(Header[" /"]: "1+3" => Int(1), Plus, Int(3)); @@ -666,33 +662,33 @@ mod tests { #[test] fn test_tokenize_idents() { // Test valid identifiers. - t!(Header[" /"]: "x" => Id("x")); - t!(Header[" /"]: "value" => Id("value")); - t!(Header[" /"]: "__main__" => Id("__main__")); - t!(Header[" /"]: "_snake_case" => Id("_snake_case")); + t!(Header[" /"]: "x" => Ident("x")); + t!(Header[" /"]: "value" => Ident("value")); + t!(Header[" /"]: "__main__" => Ident("__main__")); + t!(Header[" /"]: "_snake_case" => Ident("_snake_case")); // Test non-ascii. - t!(Header[" /"]: "Ξ±" => Id("Ξ±")); - t!(Header[" /"]: "αž˜αŸ’αžαžΆαž™" => Id("αž˜αŸ’αžαžΆαž™")); + t!(Header[" /"]: "Ξ±" => Ident("Ξ±")); + t!(Header[" /"]: "αž˜αŸ’αžαžΆαž™" => Ident("αž˜αŸ’αžαžΆαž™")); // Test hyphen parsed as identifier. - t!(Header[" /"]: "kebab-case" => Id("kebab-case")); - t!(Header[" /"]: "one-10" => Id("one-10")); + t!(Header[" /"]: "kebab-case" => Ident("kebab-case")); + t!(Header[" /"]: "one-10" => Ident("one-10")); } #[test] fn test_tokenize_keywords() { // Test none. t!(Header[" /"]: "none" => Token::None); - t!(Header[" /"]: "None" => Id("None")); + t!(Header[" /"]: "None" => Ident("None")); // Test valid bools. t!(Header[" /"]: "false" => Bool(false)); t!(Header[" /"]: "true" => Bool(true)); // Test invalid bools. - t!(Header[" /"]: "True" => Id("True")); - t!(Header[" /"]: "falser" => Id("falser")); + t!(Header[" /"]: "True" => Ident("True")); + t!(Header[" /"]: "falser" => Ident("falser")); } #[test] @@ -775,15 +771,15 @@ mod tests { fn test_tokenize_invalid() { // Test invalidly closed block comments. t!(Both: "*/" => StarSlash); - t!(Both: "/**/*/" => BC(""), StarSlash); + t!(Both: "/**/*/" => BlockComment(""), StarSlash); // Test invalid expressions. t!(Header: r"\" => Invalid(r"\")); t!(Header: "πŸŒ“" => Invalid("πŸŒ“")); t!(Header: r"\:" => Invalid(r"\"), Colon); - t!(Header: "meal⌚" => Id("meal"), Invalid("⌚")); - t!(Header[" /"]: r"\a" => Invalid(r"\"), Id("a")); - t!(Header[" /"]: ">main" => Invalid(">"), Id("main")); + t!(Header: "meal⌚" => Ident("meal"), Invalid("⌚")); + t!(Header[" /"]: r"\a" => Invalid(r"\"), Ident("a")); + t!(Header[" /"]: ">main" => Invalid(">"), Ident("main")); // Test invalid number suffixes. t!(Header[" /"]: "1foo" => Invalid("1foo")); diff --git a/src/pretty.rs b/src/pretty.rs index a7482869b..f6398495f 100644 --- a/src/pretty.rs +++ b/src/pretty.rs @@ -39,7 +39,7 @@ impl Printer { Write::write_fmt(self, fmt) } - /// Write a comma-separated list of items. + /// Write a list of items joined by a joiner. pub fn join(&mut self, items: I, joiner: &str, mut write_item: F) where I: IntoIterator, diff --git a/src/syntax/expr.rs b/src/syntax/expr.rs index 4916f34f6..ae8762092 100644 --- a/src/syntax/expr.rs +++ b/src/syntax/expr.rs @@ -5,8 +5,27 @@ use crate::geom::Unit; /// An expression. #[derive(Debug, Clone, PartialEq)] pub enum Expr { - /// A literal: `true`, `1cm`, `"hi"`. - Lit(Lit), + /// The none literal: `none`. + None, + /// A identifier literal: `left`. + Ident(Ident), + /// A boolean literal: `true`, `false`. + Bool(bool), + /// An integer literal: `120`. + Int(i64), + /// A floating-point literal: `1.2`, `10e-4`. + Float(f64), + /// A length literal: `12pt`, `3cm`. + Length(f64, Unit), + /// A percent literal: `50%`. + /// + /// _Note_: `50%` is stored as `50.0` here, but as `0.5` in the + /// corresponding [value](crate::geom::Relative). + Percent(f64), + /// A color literal: `#ffccee`. + Color(RgbaColor), + /// A string literal: `"hello!"`. + Str(String), /// An invocation of a function: `[foo ...]`, `foo(...)`. Call(ExprCall), /// A unary operation: `-x`. @@ -24,7 +43,15 @@ pub enum Expr { impl Pretty for Expr { fn pretty(&self, p: &mut Printer) { match self { - Self::Lit(lit) => lit.pretty(p), + Self::None => p.push_str("none"), + Self::Ident(v) => p.push_str(&v), + Self::Bool(v) => write!(p, "{}", v).unwrap(), + Self::Int(v) => write!(p, "{}", v).unwrap(), + Self::Float(v) => write!(p, "{}", v).unwrap(), + Self::Length(v, u) => write!(p, "{}{}", v, u).unwrap(), + Self::Percent(v) => write!(p, "{}%", v).unwrap(), + Self::Color(v) => write!(p, "{}", v).unwrap(), + Self::Str(s) => write!(p, "{:?}", &s).unwrap(), Self::Call(call) => call.pretty(p), Self::Unary(unary) => unary.pretty(p), Self::Binary(binary) => binary.pretty(p), @@ -264,48 +291,6 @@ impl Pretty for ExprDict { /// A content expression: `{*Hello* there!}`. pub type ExprContent = Tree; -/// A literal. -#[derive(Debug, Clone, PartialEq)] -pub enum Lit { - /// A identifier literal: `left`. - Ident(Ident), - /// The none literal: `none`. - None, - /// A boolean literal: `true`, `false`. - Bool(bool), - /// An integer literal: `120`. - Int(i64), - /// A floating-point literal: `1.2`, `10e-4`. - Float(f64), - /// A length literal: `12pt`, `3cm`. - Length(f64, Unit), - /// A percent literal: `50%`. - /// - /// _Note_: `50%` is stored as `50.0` here, but as `0.5` in the - /// corresponding [value](crate::geom::Relative). - Percent(f64), - /// A color literal: `#ffccee`. - Color(RgbaColor), - /// A string literal: `"hello!"`. - Str(String), -} - -impl Pretty for Lit { - fn pretty(&self, p: &mut Printer) { - match self { - Self::Ident(v) => p.push_str(&v), - Self::None => p.push_str("none"), - Self::Bool(v) => write!(p, "{}", v).unwrap(), - Self::Int(v) => write!(p, "{}", v).unwrap(), - Self::Float(v) => write!(p, "{}", v).unwrap(), - Self::Length(v, u) => write!(p, "{}{}", v, u).unwrap(), - Self::Percent(v) => write!(p, "{}%", v).unwrap(), - Self::Color(v) => write!(p, "{}", v).unwrap(), - Self::Str(s) => write!(p, "{:?}", &s).unwrap(), - } - } -} - #[cfg(test)] mod tests { use super::super::tests::test_pretty; diff --git a/src/syntax/node.rs b/src/syntax/node.rs index 91fa72d7c..01b4ee42a 100644 --- a/src/syntax/node.rs +++ b/src/syntax/node.rs @@ -5,24 +5,20 @@ use super::*; pub enum Node { /// Plain text. Text(String), - /// Whitespace containing less than two newlines. Space, /// A forced line break. Linebreak, /// A paragraph break. Parbreak, - /// Strong text was enabled / disabled. Strong, /// Emphasized text was enabled / disabled. Emph, - /// A section heading. Heading(NodeHeading), /// An optionally syntax-highlighted raw block. Raw(NodeRaw), - /// An expression. Expr(Expr), } diff --git a/src/syntax/token.rs b/src/syntax/token.rs index a28c35b6a..9a7379ca8 100644 --- a/src/syntax/token.rs +++ b/src/syntax/token.rs @@ -3,13 +3,13 @@ use crate::geom::Unit; /// A minimal semantic entity of source code. #[derive(Debug, Copy, Clone, PartialEq)] pub enum Token<'s> { + /// A consecutive non-markup string. + Text(&'s str), /// One or more whitespace characters. /// /// The contained `usize` denotes the number of newlines that were contained /// in the whitespace. Space(usize), - /// A consecutive non-markup string. - Text(&'s str), /// A line comment with inner string contents `//\n`. LineComment(&'s str), @@ -20,6 +20,19 @@ pub enum Token<'s> { /// An end of a block comment that was not started. StarSlash, + /// A left bracket: `[`. + LeftBracket, + /// A right bracket: `]`. + RightBracket, + /// A left brace: `{`. + LeftBrace, + /// A right brace: `}`. + RightBrace, + /// A left parenthesis: `(`. + LeftParen, + /// A right parenthesis: `)`. + RightParen, + /// A star: `*`. Star, /// An underscore: `_`. @@ -35,19 +48,6 @@ pub enum Token<'s> { /// A unicode escape sequence: `\u{1F5FA}`. UnicodeEscape(TokenUnicodeEscape<'s>), - /// A left bracket: `[`. - LeftBracket, - /// A right bracket: `]`. - RightBracket, - /// A left brace: `{`. - LeftBrace, - /// A right brace: `}`. - RightBrace, - /// A left parenthesis: `(`. - LeftParen, - /// A right parenthesis: `)`. - RightParen, - /// A colon: `:`. Colon, /// A comma: `,`. @@ -76,7 +76,7 @@ pub enum Token<'s> { /// A percentage: `50%`. /// /// _Note_: `50%` is stored as `50.0` here, as in the corresponding - /// [literal](super::Lit::Percent). + /// [literal](super::Expr::Percent). Percent(f64), /// A hex value: `#20d82a`. Hex(&'s str), @@ -124,13 +124,20 @@ impl<'s> Token<'s> { /// The natural-language name of this token for use in error messages. pub fn name(self) -> &'static str { match self { - Self::Space(_) => "space", Self::Text(_) => "text", + Self::Space(_) => "space", Self::LineComment(_) => "line comment", Self::BlockComment(_) => "block comment", Self::StarSlash => "end of block comment", + Self::LeftBracket => "opening bracket", + Self::RightBracket => "closing bracket", + Self::LeftBrace => "opening brace", + Self::RightBrace => "closing brace", + Self::LeftParen => "opening paren", + Self::RightParen => "closing paren", + Self::Star => "star", Self::Underscore => "underscore", Self::Backslash => "backslash", @@ -139,13 +146,6 @@ impl<'s> Token<'s> { Self::Raw { .. } => "raw block", Self::UnicodeEscape { .. } => "unicode escape sequence", - Self::LeftBracket => "opening bracket", - Self::RightBracket => "closing bracket", - Self::LeftBrace => "opening brace", - Self::RightBrace => "closing brace", - Self::LeftParen => "opening paren", - Self::RightParen => "closing paren", - Self::Colon => "colon", Self::Comma => "comma", Self::Pipe => "pipe", @@ -153,8 +153,8 @@ impl<'s> Token<'s> { Self::Hyphen => "minus sign", Self::Slash => "slash", - Self::Ident(_) => "identifier", Self::None => "none", + Self::Ident(_) => "identifier", Self::Bool(_) => "bool", Self::Int(_) => "integer", Self::Float(_) => "float",