Tidy up AST

This commit is contained in:
Laurenz 2021-11-07 22:05:48 +01:00
parent 8117ca9950
commit 95866d5fc9
10 changed files with 370 additions and 561 deletions

View File

@ -1,93 +1,27 @@
use std::rc::Rc; use std::rc::Rc;
use super::{Scope, Scopes, Value}; use super::{Scope, Scopes};
use crate::syntax::ast::{ClosureParam, Expr, Imports}; use crate::syntax::{NodeKind, RedRef};
use crate::syntax::RedRef;
/// A visitor that captures variable slots. /// A visitor that captures variable slots.
pub struct CapturesVisitor<'a> { pub struct CapturesVisitor<'a> {
external: &'a Scopes<'a>, external: &'a Scopes<'a>,
internal: Scopes<'a>,
captures: Scope, captures: Scope,
} }
impl<'a> CapturesVisitor<'a> { impl<'a> CapturesVisitor<'a> {
/// Create a new visitor for the given external scopes. /// Create a new visitor for the given external scopes.
pub fn new(external: &'a Scopes) -> Self { pub fn new(external: &'a Scopes) -> Self {
Self { Self { external, captures: Scope::new() }
external,
internal: Scopes::new(None),
captures: Scope::new(),
}
} }
pub fn visit(&mut self, node: RedRef) { pub fn visit(&mut self, node: RedRef) {
let expr: Option<Expr> = node.cast(); match node.kind() {
NodeKind::Ident(ident) => {
match expr.as_ref() { if let Some(slot) = self.external.get(ident.as_str()) {
Some(Expr::Let(expr)) => { self.captures.def_slot(ident.as_str(), Rc::clone(slot));
self.visit(expr.init_ref());
let ident = expr.binding();
self.internal.def_mut(ident.as_str(), Value::None);
}
Some(Expr::Closure(closure)) => {
for arg in closure.params() {
match arg {
ClosureParam::Pos(ident) | ClosureParam::Sink(ident) => {
self.internal.def_mut(ident.as_str(), Value::None);
}
ClosureParam::Named(name) => {
self.internal.def_mut(name.name().as_str(), Value::None);
}
}
}
self.visit(closure.body_ref());
}
Some(Expr::For(forloop)) => {
let pattern = forloop.pattern();
self.internal.def_mut(pattern.value().as_str(), Value::None);
if let Some(key) = pattern.key() {
self.internal.def_mut(key.as_str(), Value::None);
}
self.visit(forloop.body_ref());
}
Some(Expr::Import(import)) => {
if let Imports::Idents(idents) = import.imports() {
for ident in idents {
self.internal.def_mut(ident.as_str(), Value::None);
}
} }
} }
Some(Expr::Ident(ident)) => {
if self.internal.get(ident.as_str()).is_none() {
if let Some(slot) = self.external.get(ident.as_str()) {
self.captures.def_slot(ident.as_str(), Rc::clone(slot));
}
}
}
_ => {}
}
match expr.as_ref() {
Some(Expr::Let(_)) | Some(Expr::For(_)) | Some(Expr::Closure(_)) => {}
Some(Expr::Block(_)) => {
self.internal.enter();
for child in node.children() {
self.visit(child);
}
self.internal.exit();
}
Some(Expr::Template(_)) => {
self.internal.enter();
for child in node.children() {
self.visit(child);
}
self.internal.exit();
}
_ => { _ => {
for child in node.children() { for child in node.children() {
self.visit(child); self.visit(child);

View File

@ -219,7 +219,7 @@ impl Eval for Ident {
type Output = Value; type Output = Value;
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> { fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
match ctx.scopes.get(self) { match ctx.scopes.get(&self.string) {
Some(slot) => Ok(slot.borrow().clone()), Some(slot) => Ok(slot.borrow().clone()),
None => bail!(self.span, "unknown variable"), None => bail!(self.span, "unknown variable"),
} }
@ -401,7 +401,7 @@ impl Eval for CallArgs {
CallArg::Named(x) => { CallArg::Named(x) => {
items.push(Arg { items.push(Arg {
span, span,
name: Some((&x.name().string).into()), name: Some(x.name().string.into()),
value: Spanned::new(x.expr().eval(ctx)?, x.expr().span()), value: Spanned::new(x.expr().eval(ctx)?, x.expr().span()),
}); });
} }
@ -443,12 +443,10 @@ impl Eval for ClosureExpr {
type Output = Value; type Output = Value;
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> { fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
let name = self.name().as_ref().map(|name| name.string.clone());
// Collect captured variables. // Collect captured variables.
let captured = { let captured = {
let mut visitor = CapturesVisitor::new(&ctx.scopes); let mut visitor = CapturesVisitor::new(&ctx.scopes);
visitor.visit(self.underlying()); visitor.visit(self.as_red());
visitor.finish() visitor.finish()
}; };
@ -459,23 +457,24 @@ impl Eval for ClosureExpr {
for param in self.params() { for param in self.params() {
match param { match param {
ClosureParam::Pos(name) => { ClosureParam::Pos(name) => {
params.push((name.string.clone(), None)); params.push((name.string, None));
} }
ClosureParam::Named(x) => { ClosureParam::Named(named) => {
params.push((x.name().string.clone(), Some(x.expr().eval(ctx)?))); params.push((named.name().string, Some(named.expr().eval(ctx)?)));
} }
ClosureParam::Sink(name) => { ClosureParam::Sink(name) => {
if sink.is_some() { if sink.is_some() {
bail!(name.span, "only one argument sink is allowed"); bail!(name.span, "only one argument sink is allowed");
} }
sink = Some(name.string.clone()); sink = Some(name.string);
} }
} }
} }
// Clone the body expression so that we don't have a lifetime // Clone the body expression so that we don't have a lifetime
// dependence on the AST. // dependence on the AST.
let body = self.body().clone(); let name = self.name().map(|name| name.string);
let body = self.body();
// Define the actual function. // Define the actual function.
let func = Function::new(name, move |ctx, args| { let func = Function::new(name, move |ctx, args| {
@ -534,7 +533,7 @@ impl Eval for LetExpr {
Some(expr) => expr.eval(ctx)?, Some(expr) => expr.eval(ctx)?,
None => Value::None, None => Value::None,
}; };
ctx.scopes.def_mut(self.binding().as_str(), value); ctx.scopes.def_mut(self.binding().string, value);
Ok(Value::None) Ok(Value::None)
} }
} }
@ -590,7 +589,7 @@ impl Eval for ForExpr {
#[allow(unused_parens)] #[allow(unused_parens)]
for ($($value),*) in $iter { for ($($value),*) in $iter {
$(ctx.scopes.def_mut($binding.as_str(), $value);)* $(ctx.scopes.def_mut(&$binding.string, $value);)*
let value = self.body().eval(ctx)?; let value = self.body().eval(ctx)?;
output = ops::join(output, value) output = ops::join(output, value)
@ -637,16 +636,16 @@ impl Eval for ImportExpr {
let file = ctx.import(&path, self.path().span())?; let file = ctx.import(&path, self.path().span())?;
let module = &ctx.modules[&file]; let module = &ctx.modules[&file];
match &self.imports() { match self.imports() {
Imports::Wildcard => { Imports::Wildcard => {
for (var, slot) in module.scope.iter() { for (var, slot) in module.scope.iter() {
ctx.scopes.def_mut(var, slot.borrow().clone()); ctx.scopes.def_mut(var, slot.borrow().clone());
} }
} }
Imports::Idents(idents) => { Imports::Items(idents) => {
for ident in idents { for ident in idents {
if let Some(slot) = module.scope.get(&ident) { if let Some(slot) = module.scope.get(&ident.string) {
ctx.scopes.def_mut(ident.as_str(), slot.borrow().clone()); ctx.scopes.def_mut(ident.string, slot.borrow().clone());
} else { } else {
bail!(ident.span, "unresolved import"); bail!(ident.span, "unresolved import");
} }
@ -692,7 +691,7 @@ impl Access for Expr {
impl Access for Ident { impl Access for Ident {
fn access<'a>(&self, ctx: &'a mut EvalContext) -> TypResult<RefMut<'a, Value>> { fn access<'a>(&self, ctx: &'a mut EvalContext) -> TypResult<RefMut<'a, Value>> {
match ctx.scopes.get(self) { match ctx.scopes.get(&self.string) {
Some(slot) => match slot.try_borrow_mut() { Some(slot) => match slot.try_borrow_mut() {
Ok(guard) => Ok(guard), Ok(guard) => Ok(guard),
Err(_) => bail!(self.span, "cannot mutate a constant"), Err(_) => bail!(self.span, "cannot mutate a constant"),

View File

@ -13,7 +13,7 @@ pub use tokens::*;
use std::rc::Rc; use std::rc::Rc;
use crate::syntax::ast::{Associativity, BinOp, UnOp}; use crate::syntax::ast::{Associativity, BinOp, UnOp};
use crate::syntax::{ErrorPosition, Green, GreenNode, NodeKind}; use crate::syntax::{ErrorPos, Green, GreenNode, NodeKind};
/// Parse a source file. /// Parse a source file.
pub fn parse(source: &str) -> Rc<GreenNode> { pub fn parse(source: &str) -> Rc<GreenNode> {
@ -197,7 +197,7 @@ fn expr_prec(p: &mut Parser, atomic: bool, min_prec: usize) -> ParseResult {
p.peek_direct(), p.peek_direct(),
Some(NodeKind::LeftParen | NodeKind::LeftBracket) Some(NodeKind::LeftParen | NodeKind::LeftBracket)
) { ) {
call(p, &marker)?; call(p, marker)?;
continue; continue;
} }
@ -206,7 +206,7 @@ fn expr_prec(p: &mut Parser, atomic: bool, min_prec: usize) -> ParseResult {
} }
if p.at(&NodeKind::With) { if p.at(&NodeKind::With) {
with_expr(p, &marker)?; with_expr(p, marker)?;
} }
let op = match p.peek().and_then(BinOp::from_token) { let op = match p.peek().and_then(BinOp::from_token) {
@ -248,7 +248,7 @@ fn primary(p: &mut Parser, atomic: bool) -> ParseResult {
// Arrow means this is a closure's lone parameter. // Arrow means this is a closure's lone parameter.
if !atomic && p.at(&NodeKind::Arrow) { if !atomic && p.at(&NodeKind::Arrow) {
marker.end(p, NodeKind::ClosureParams); marker.end(p, NodeKind::ClosureParams);
p.eat(); p.eat_assert(&NodeKind::Arrow);
marker.perform(p, NodeKind::Closure, expr) marker.perform(p, NodeKind::Closure, expr)
} else { } else {
Ok(()) Ok(())
@ -326,14 +326,13 @@ fn parenthesized(p: &mut Parser) -> ParseResult {
// Leading colon makes this a (empty) dictionary. // Leading colon makes this a (empty) dictionary.
if colon { if colon {
dict(p, &marker); dict(p, marker);
return Ok(()); return Ok(());
} }
// Arrow means this is a closure's parameter list. // Arrow means this is a closure's parameter list.
if p.at(&NodeKind::Arrow) { if p.at(&NodeKind::Arrow) {
params(p, &marker, true); params(p, marker);
marker.end(p, NodeKind::ClosureParams);
p.eat_assert(&NodeKind::Arrow); p.eat_assert(&NodeKind::Arrow);
return marker.perform(p, NodeKind::Closure, expr); return marker.perform(p, NodeKind::Closure, expr);
} }
@ -341,8 +340,8 @@ fn parenthesized(p: &mut Parser) -> ParseResult {
// Transform into the identified collection. // Transform into the identified collection.
match kind { match kind {
CollectionKind::Group => marker.end(p, NodeKind::Group), CollectionKind::Group => marker.end(p, NodeKind::Group),
CollectionKind::Positional => array(p, &marker), CollectionKind::Positional => array(p, marker),
CollectionKind::Named => dict(p, &marker), CollectionKind::Named => dict(p, marker),
} }
Ok(()) Ok(())
@ -384,7 +383,7 @@ fn collection(p: &mut Parser) -> (CollectionKind, usize) {
items += 1; items += 1;
if let Some(marker) = missing_coma.take() { if let Some(marker) = missing_coma.take() {
marker.expected_at(p, "comma"); marker.expected(p, "comma");
} }
if p.eof() { if p.eof() {
@ -419,12 +418,11 @@ fn item(p: &mut Parser) -> ParseResult<NodeKind> {
if p.at(&NodeKind::Colon) { if p.at(&NodeKind::Colon) {
marker.perform(p, NodeKind::Named, |p| { marker.perform(p, NodeKind::Named, |p| {
if matches!(marker.child_at(p).unwrap().kind(), &NodeKind::Ident(_)) { if matches!(marker.peek(p).unwrap().kind(), &NodeKind::Ident(_)) {
p.eat(); p.eat();
expr(p) expr(p)
} else { } else {
let error = let error = NodeKind::Error(ErrorPos::Full, "expected identifier".into());
NodeKind::Error(ErrorPosition::Full, "expected identifier".into());
marker.end(p, error); marker.end(p, error);
p.eat(); p.eat();
expr(p).ok(); expr(p).ok();
@ -440,15 +438,10 @@ fn item(p: &mut Parser) -> ParseResult<NodeKind> {
/// Convert a collection into an array, producing errors for anything other than /// Convert a collection into an array, producing errors for anything other than
/// expressions. /// expressions.
fn array(p: &mut Parser, marker: &Marker) { fn array(p: &mut Parser, marker: Marker) {
marker.filter_children(p, |x| match x.kind() { marker.filter_children(p, |x| match x.kind() {
NodeKind::Named => Err(( NodeKind::Named => Err("expected expression, found named pair"),
ErrorPosition::Full, NodeKind::Spread => Err("spreading is not allowed here"),
"expected expression, found named pair".into(),
)),
NodeKind::Spread => {
Err((ErrorPosition::Full, "spreading is not allowed here".into()))
}
_ => Ok(()), _ => Ok(()),
}); });
marker.end(p, NodeKind::Array); marker.end(p, NodeKind::Array);
@ -456,25 +449,21 @@ fn array(p: &mut Parser, marker: &Marker) {
/// Convert a collection into a dictionary, producing errors for anything other /// Convert a collection into a dictionary, producing errors for anything other
/// than named pairs. /// than named pairs.
fn dict(p: &mut Parser, marker: &Marker) { fn dict(p: &mut Parser, marker: Marker) {
marker.filter_children(p, |x| match x.kind() { marker.filter_children(p, |x| match x.kind() {
kind if kind.is_paren() => Ok(()),
NodeKind::Named | NodeKind::Comma | NodeKind::Colon => Ok(()), NodeKind::Named | NodeKind::Comma | NodeKind::Colon => Ok(()),
NodeKind::Spread => { NodeKind::Spread => Err("spreading is not allowed here"),
Err((ErrorPosition::Full, "spreading is not allowed here".into())) _ => Err("expected named pair, found expression"),
}
_ if x.kind().is_paren() => Ok(()),
_ => Err((
ErrorPosition::Full,
"expected named pair, found expression".into(),
)),
}); });
marker.end(p, NodeKind::Dict); marker.end(p, NodeKind::Dict);
} }
/// Convert a collection into a list of parameters, producing errors for /// Convert a collection into a list of parameters, producing errors for
/// anything other than identifiers, spread operations and named pairs. /// anything other than identifiers, spread operations and named pairs.
fn params(p: &mut Parser, marker: &Marker, allow_parens: bool) { fn params(p: &mut Parser, marker: Marker) {
marker.filter_children(p, |x| match x.kind() { marker.filter_children(p, |x| match x.kind() {
kind if kind.is_paren() => Ok(()),
NodeKind::Named | NodeKind::Comma | NodeKind::Ident(_) => Ok(()), NodeKind::Named | NodeKind::Comma | NodeKind::Ident(_) => Ok(()),
NodeKind::Spread NodeKind::Spread
if matches!( if matches!(
@ -484,9 +473,9 @@ fn params(p: &mut Parser, marker: &Marker, allow_parens: bool) {
{ {
Ok(()) Ok(())
} }
_ if allow_parens && x.kind().is_paren() => Ok(()), _ => Err("expected identifier"),
_ => Err((ErrorPosition::Full, "expected identifier".into())),
}); });
marker.end(p, NodeKind::ClosureParams);
} }
// Parse a template block: `[...]`. // Parse a template block: `[...]`.
@ -517,7 +506,7 @@ fn block(p: &mut Parser) {
} }
/// Parse a function call. /// Parse a function call.
fn call(p: &mut Parser, callee: &Marker) -> ParseResult { fn call(p: &mut Parser, callee: Marker) -> ParseResult {
callee.perform(p, NodeKind::Call, |p| match p.peek_direct() { callee.perform(p, NodeKind::Call, |p| match p.peek_direct() {
Some(NodeKind::LeftParen | NodeKind::LeftBracket) => { Some(NodeKind::LeftParen | NodeKind::LeftBracket) => {
args(p, true); args(p, true);
@ -546,7 +535,7 @@ fn args(p: &mut Parser, allow_template: bool) {
} }
/// Parse a with expression. /// Parse a with expression.
fn with_expr(p: &mut Parser, marker: &Marker) -> ParseResult { fn with_expr(p: &mut Parser, marker: Marker) -> ParseResult {
marker.perform(p, NodeKind::WithExpr, |p| { marker.perform(p, NodeKind::WithExpr, |p| {
p.eat_assert(&NodeKind::With); p.eat_assert(&NodeKind::With);
@ -569,18 +558,16 @@ fn let_expr(p: &mut Parser) -> ParseResult {
ident(p)?; ident(p)?;
if p.at(&NodeKind::With) { if p.at(&NodeKind::With) {
with_expr(p, &marker)?; with_expr(p, marker)?;
} else { } else {
// If a parenthesis follows, this is a function definition. // If a parenthesis follows, this is a function definition.
let has_params = p.peek_direct() == Some(&NodeKind::LeftParen); let has_params = p.peek_direct() == Some(&NodeKind::LeftParen);
if has_params { if has_params {
p.perform(NodeKind::ClosureParams, |p| { let marker = p.marker();
p.start_group(Group::Paren); p.start_group(Group::Paren);
let marker = p.marker(); collection(p);
collection(p); p.end_group();
params(p, &marker, true); params(p, marker);
p.end_group();
});
} }
if p.eat_if(&NodeKind::Eq) { if p.eat_if(&NodeKind::Eq) {
@ -671,7 +658,7 @@ fn import_expr(p: &mut Parser) -> ParseResult {
marker.filter_children(p, |n| match n.kind() { marker.filter_children(p, |n| match n.kind() {
NodeKind::Ident(_) | NodeKind::Comma => Ok(()), NodeKind::Ident(_) | NodeKind::Comma => Ok(()),
_ => Err((ErrorPosition::Full, "expected identifier".into())), _ => Err("expected identifier"),
}); });
}); });
}; };

View File

@ -1,7 +1,7 @@
use std::mem; use std::mem;
use super::{TokenMode, Tokens}; use super::{TokenMode, Tokens};
use crate::syntax::{ErrorPosition, Green, GreenData, GreenNode, NodeKind}; use crate::syntax::{ErrorPos, Green, GreenData, GreenNode, NodeKind};
use crate::util::EcoString; use crate::util::EcoString;
/// Allows parser methods to use the try operator. Not exposed as the parser /// Allows parser methods to use the try operator. Not exposed as the parser
@ -131,11 +131,9 @@ impl<'s> Parser<'s> {
/// Eat the current token, but change its type. /// Eat the current token, but change its type.
pub fn convert(&mut self, kind: NodeKind) { pub fn convert(&mut self, kind: NodeKind) {
let idx = self.children.len(); let marker = self.marker();
self.eat(); self.eat();
if let Some(child) = self.children.get_mut(idx) { marker.convert(self, kind);
child.set_kind(kind);
}
} }
/// Whether the current token is of the given type. /// Whether the current token is of the given type.
@ -321,7 +319,7 @@ impl<'s> Parser<'s> {
impl Parser<'_> { impl Parser<'_> {
/// Push an error into the children list. /// Push an error into the children list.
pub fn push_error(&mut self, msg: impl Into<EcoString>) { pub fn push_error(&mut self, msg: impl Into<EcoString>) {
let error = NodeKind::Error(ErrorPosition::Full, msg.into()); let error = NodeKind::Error(ErrorPos::Full, msg.into());
self.children.push(GreenData::new(error, 0).into()); self.children.push(GreenData::new(error, 0).into());
} }
@ -330,7 +328,7 @@ impl Parser<'_> {
match self.peek() { match self.peek() {
Some(found) => { Some(found) => {
let msg = format!("unexpected {}", found); let msg = format!("unexpected {}", found);
let error = NodeKind::Error(ErrorPosition::Full, msg.into()); let error = NodeKind::Error(ErrorPos::Full, msg.into());
self.perform(error, Self::eat); self.perform(error, Self::eat);
} }
None => self.push_error("unexpected end of file"), None => self.push_error("unexpected end of file"),
@ -342,7 +340,7 @@ impl Parser<'_> {
match self.peek() { match self.peek() {
Some(found) => { Some(found) => {
let msg = format!("expected {}, found {}", thing, found); let msg = format!("expected {}, found {}", thing, found);
let error = NodeKind::Error(ErrorPosition::Full, msg.into()); let error = NodeKind::Error(ErrorPos::Full, msg.into());
self.perform(error, Self::eat); self.perform(error, Self::eat);
} }
None => self.expected_at(thing), None => self.expected_at(thing),
@ -352,7 +350,7 @@ impl Parser<'_> {
/// Add an error that the `thing` was expected at the end of the last /// Add an error that the `thing` was expected at the end of the last
/// non-trivia token. /// non-trivia token.
pub fn expected_at(&mut self, thing: &str) { pub fn expected_at(&mut self, thing: &str) {
Marker(self.trivia_start()).expected_at(self, thing); Marker(self.trivia_start()).expected(self, thing);
} }
} }
@ -384,15 +382,15 @@ impl Marker {
/// Wrap all children that do not fulfill the predicate in error nodes. /// Wrap all children that do not fulfill the predicate in error nodes.
pub fn filter_children<F>(self, p: &mut Parser, f: F) pub fn filter_children<F>(self, p: &mut Parser, f: F)
where where
F: Fn(&Green) -> Result<(), (ErrorPosition, EcoString)>, F: Fn(&Green) -> Result<(), &'static str>,
{ {
for child in &mut p.children[self.0 ..] { for child in &mut p.children[self.0 ..] {
if (p.tokens.mode() == TokenMode::Markup if (p.tokens.mode() == TokenMode::Markup
|| !Parser::is_trivia_ext(child.kind(), false)) || !Parser::is_trivia_ext(child.kind(), false))
&& !child.kind().is_error() && !child.kind().is_error()
{ {
if let Err((pos, msg)) = f(child) { if let Err(msg) = f(child) {
let error = NodeKind::Error(pos, msg); let error = NodeKind::Error(ErrorPos::Full, msg.into());
let inner = mem::take(child); let inner = mem::take(child);
*child = GreenNode::with_child(error, inner).into(); *child = GreenNode::with_child(error, inner).into();
} }
@ -401,16 +399,23 @@ impl Marker {
} }
/// Insert an error message that `what` was expected at the marker position. /// Insert an error message that `what` was expected at the marker position.
pub fn expected_at(self, p: &mut Parser, what: &str) { pub fn expected(self, p: &mut Parser, what: &str) {
let msg = format!("expected {}", what); let msg = format!("expected {}", what);
let error = NodeKind::Error(ErrorPosition::Full, msg.into()); let error = NodeKind::Error(ErrorPos::Full, msg.into());
p.children.insert(self.0, GreenData::new(error, 0).into()); p.children.insert(self.0, GreenData::new(error, 0).into());
} }
/// Return a reference to the child directly after the marker. /// Peek at the child directly after the marker.
pub fn child_at<'a>(self, p: &'a Parser) -> Option<&'a Green> { pub fn peek<'a>(self, p: &'a Parser) -> Option<&'a Green> {
p.children.get(self.0) p.children.get(self.0)
} }
/// Convert the child directly after marker.
pub fn convert(self, p: &mut Parser, kind: NodeKind) {
if let Some(child) = p.children.get_mut(self.0) {
child.convert(kind);
}
}
} }
/// A logical group of tokens, e.g. `[...]`. /// A logical group of tokens, e.g. `[...]`.

View File

@ -1,4 +1,4 @@
use super::{is_newline, Scanner}; use super::{is_ident, is_newline, Scanner};
use crate::syntax::RawData; use crate::syntax::RawData;
use crate::util::EcoString; use crate::util::EcoString;
@ -51,7 +51,7 @@ pub fn resolve_raw(column: usize, backticks: u8, text: &str) -> RawData {
let (tag, inner) = split_at_lang_tag(text); let (tag, inner) = split_at_lang_tag(text);
let (text, block) = trim_and_split_raw(column, inner); let (text, block) = trim_and_split_raw(column, inner);
RawData { RawData {
lang: Some(tag.into()), lang: is_ident(tag).then(|| tag.into()),
text: text.into(), text: text.into(),
backticks, backticks,
block, block,
@ -201,15 +201,15 @@ mod tests {
// More than one backtick with lang tag. // More than one backtick with lang tag.
test(0, 2, "js alert()", Some("js"), "alert()", false); test(0, 2, "js alert()", Some("js"), "alert()", false);
test(0, 3, "py quit(\n\n)", Some("py"), "quit(\n\n)", true); test(0, 3, "py quit(\n\n)", Some("py"), "quit(\n\n)", true);
test(0, 2, "", Some(""), "", false); test(0, 2, "", None, "", false);
// Trimming of whitespace (tested more thoroughly in separate test). // Trimming of whitespace (tested more thoroughly in separate test).
test(0, 2, " a", Some(""), "a", false); test(0, 2, " a", None, "a", false);
test(0, 2, " a", Some(""), " a", false); test(0, 2, " a", None, " a", false);
test(0, 2, " \na", Some(""), "a", true); test(0, 2, " \na", None, "a", true);
// Dedenting // Dedenting
test(2, 3, " def foo():\n bar()", Some(""), "def foo():\n bar()", true); test(2, 3, " def foo():\n bar()", None, "def foo():\n bar()", true);
} }
#[test] #[test]

View File

@ -182,7 +182,13 @@ pub fn is_newline(character: char) -> bool {
) )
} }
/// Whether a string is a valid identifier. /// Whether a string is a valid unicode identifier.
///
/// In addition to what is specified in the [Unicode Standard][uax31], we allow:
/// - `_` as a starting character,
/// - `_` and `-` as continuing characters.
///
/// [uax31]: http://www.unicode.org/reports/tr31/
#[inline] #[inline]
pub fn is_ident(string: &str) -> bool { pub fn is_ident(string: &str) -> bool {
let mut chars = string.chars(); let mut chars = string.chars();

View File

@ -236,20 +236,19 @@ impl<'s> Tokens<'s> {
'u' if self.s.rest().starts_with("u{") => { 'u' if self.s.rest().starts_with("u{") => {
self.s.eat_assert('u'); self.s.eat_assert('u');
self.s.eat_assert('{'); self.s.eat_assert('{');
let sequence: EcoString = self.s.eat_while(|c| c.is_ascii_alphanumeric()).into(); let sequence = self.s.eat_while(|c| c.is_ascii_alphanumeric());
if self.s.eat_if('}') { if self.s.eat_if('}') {
if let Some(c) = resolve_hex(&sequence) { if let Some(c) = resolve_hex(&sequence) {
NodeKind::UnicodeEscape(c) NodeKind::UnicodeEscape(c)
} else { } else {
NodeKind::Error( NodeKind::Error(
ErrorPosition::Full, ErrorPos::Full,
"invalid unicode escape sequence".into(), "invalid unicode escape sequence".into(),
) )
} }
} else { } else {
NodeKind::Error( NodeKind::Error(
ErrorPosition::End, ErrorPos::End,
"expected closing brace".into(), "expected closing brace".into(),
) )
} }
@ -348,7 +347,7 @@ impl<'s> Tokens<'s> {
let noun = if remaining == 1 { "backtick" } else { "backticks" }; let noun = if remaining == 1 { "backtick" } else { "backticks" };
NodeKind::Error( NodeKind::Error(
ErrorPosition::End, ErrorPos::End,
if found == 0 { if found == 0 {
format!("expected {} {}", remaining, noun) format!("expected {} {}", remaining, noun)
} else { } else {
@ -396,7 +395,7 @@ impl<'s> Tokens<'s> {
})) }))
} else { } else {
NodeKind::Error( NodeKind::Error(
ErrorPosition::End, ErrorPos::End,
if !display || (!escaped && dollar) { if !display || (!escaped && dollar) {
"expected closing dollar sign" "expected closing dollar sign"
} else { } else {
@ -487,7 +486,7 @@ impl<'s> Tokens<'s> {
if self.s.eat_if('"') { if self.s.eat_if('"') {
NodeKind::Str(string) NodeKind::Str(string)
} else { } else {
NodeKind::Error(ErrorPosition::End, "expected quote".into()) NodeKind::Error(ErrorPos::End, "expected quote".into())
} }
} }
@ -555,7 +554,7 @@ mod tests {
use super::*; use super::*;
use ErrorPosition::*; use ErrorPos::*;
use NodeKind::*; use NodeKind::*;
use Option::None; use Option::None;
use TokenMode::{Code, Markup}; use TokenMode::{Code, Markup};
@ -564,7 +563,7 @@ mod tests {
NodeKind::UnicodeEscape(c) NodeKind::UnicodeEscape(c)
} }
fn Error(pos: ErrorPosition, message: &str) -> NodeKind { fn Error(pos: ErrorPos, message: &str) -> NodeKind {
NodeKind::Error(pos, message.into()) NodeKind::Error(pos, message.into())
} }
@ -881,7 +880,7 @@ mod tests {
// Test more backticks. // Test more backticks.
t!(Markup: "``nope``" => Raw("", None, 1, false), Text("nope"), Raw("", None, 1, false)); t!(Markup: "``nope``" => Raw("", None, 1, false), Text("nope"), Raw("", None, 1, false));
t!(Markup: "````🚀````" => Raw("", Some("🚀"), 4, false)); t!(Markup: "````🚀````" => Raw("", None, 4, false));
t!(Markup[""]: "`````👩‍🚀````noend" => Error(End, "expected 5 backticks")); t!(Markup[""]: "`````👩‍🚀````noend" => Error(End, "expected 5 backticks"));
t!(Markup[""]: "````raw``````" => Raw("", Some("raw"), 4, false), Raw("", None, 1, false)); t!(Markup[""]: "````raw``````" => Raw("", Some("raw"), 4, false), Raw("", None, 1, false));
} }

View File

@ -1,10 +1,7 @@
//! A typed layer over the red-green tree. //! A typed layer over the red-green tree.
use std::ops::Deref;
use super::{NodeKind, RedNode, RedRef, Span}; use super::{NodeKind, RedNode, RedRef, Span};
use crate::geom::{AngularUnit, LengthUnit}; use crate::geom::{AngularUnit, LengthUnit};
use crate::parse::is_ident;
use crate::util::EcoString; use crate::util::EcoString;
/// A typed AST node. /// A typed AST node.
@ -40,7 +37,7 @@ macro_rules! node {
} }
/// The underlying red node. /// The underlying red node.
pub fn underlying(&self) -> RedRef { pub fn as_red(&self) -> RedRef {
self.0.as_ref() self.0.as_ref()
} }
} }
@ -112,7 +109,7 @@ impl TypedNode for MarkupNode {
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub struct RawNode { pub struct RawNode {
/// An optional identifier specifying the language to syntax-highlight in. /// An optional identifier specifying the language to syntax-highlight in.
pub lang: Option<Ident>, pub lang: Option<EcoString>,
/// The raw text, determined as the raw string between the backticks trimmed /// The raw text, determined as the raw string between the backticks trimmed
/// according to the above rules. /// according to the above rules.
pub text: EcoString, pub text: EcoString,
@ -124,18 +121,11 @@ pub struct RawNode {
impl TypedNode for RawNode { impl TypedNode for RawNode {
fn from_red(node: RedRef) -> Option<Self> { fn from_red(node: RedRef) -> Option<Self> {
match node.kind() { match node.kind() {
NodeKind::Raw(raw) => { NodeKind::Raw(raw) => Some(Self {
let full = node.span(); block: raw.block,
let start = full.start + raw.backticks as usize; lang: raw.lang.clone(),
Some(Self { text: raw.text.clone(),
block: raw.block, }),
lang: raw.lang.as_ref().and_then(|lang| {
let span = Span::new(full.source, start, start + lang.len());
Ident::new(lang, span)
}),
text: raw.text.clone(),
})
}
_ => None, _ => None,
} }
} }
@ -149,9 +139,7 @@ node! {
impl HeadingNode { impl HeadingNode {
/// The contents of the heading. /// The contents of the heading.
pub fn body(&self) -> Markup { pub fn body(&self) -> Markup {
self.0 self.0.cast_first_child().expect("heading is missing markup body")
.cast_first_child()
.expect("heading node is missing markup body")
} }
/// The section depth (numer of equals signs). /// The section depth (numer of equals signs).
@ -184,7 +172,7 @@ node! {
impl EnumNode { impl EnumNode {
/// The contents of the list item. /// The contents of the list item.
pub fn body(&self) -> Markup { pub fn body(&self) -> Markup {
self.0.cast_first_child().expect("enumeration node is missing body") self.0.cast_first_child().expect("enum node is missing body")
} }
/// The number, if any. /// The number, if any.
@ -195,7 +183,7 @@ impl EnumNode {
NodeKind::EnumNumbering(num) => Some(num.clone()), NodeKind::EnumNumbering(num) => Some(num.clone()),
_ => None, _ => None,
}) })
.expect("enumeration node is missing number") .expect("enum node is missing number")
} }
} }
@ -240,6 +228,31 @@ pub enum Expr {
Include(IncludeExpr), Include(IncludeExpr),
} }
impl TypedNode for Expr {
fn from_red(node: RedRef) -> Option<Self> {
match node.kind() {
NodeKind::Ident(_) => node.cast().map(Self::Ident),
NodeKind::Array => node.cast().map(Self::Array),
NodeKind::Dict => node.cast().map(Self::Dict),
NodeKind::Template => node.cast().map(Self::Template),
NodeKind::Group => node.cast().map(Self::Group),
NodeKind::Block => node.cast().map(Self::Block),
NodeKind::Unary => node.cast().map(Self::Unary),
NodeKind::Binary => node.cast().map(Self::Binary),
NodeKind::Call => node.cast().map(Self::Call),
NodeKind::Closure => node.cast().map(Self::Closure),
NodeKind::WithExpr => node.cast().map(Self::With),
NodeKind::LetExpr => node.cast().map(Self::Let),
NodeKind::IfExpr => node.cast().map(Self::If),
NodeKind::WhileExpr => node.cast().map(Self::While),
NodeKind::ForExpr => node.cast().map(Self::For),
NodeKind::ImportExpr => node.cast().map(Self::Import),
NodeKind::IncludeExpr => node.cast().map(Self::Include),
_ => node.cast().map(Self::Lit),
}
}
}
impl Expr { impl Expr {
/// Whether the expression can be shortened in markup with a hashtag. /// Whether the expression can be shortened in markup with a hashtag.
pub fn has_short_form(&self) -> bool { pub fn has_short_form(&self) -> bool {
@ -280,31 +293,6 @@ impl Expr {
} }
} }
impl TypedNode for Expr {
fn from_red(node: RedRef) -> Option<Self> {
match node.kind() {
NodeKind::Ident(_) => node.cast().map(Self::Ident),
NodeKind::Array => node.cast().map(Self::Array),
NodeKind::Dict => node.cast().map(Self::Dict),
NodeKind::Template => node.cast().map(Self::Template),
NodeKind::Group => node.cast().map(Self::Group),
NodeKind::Block => node.cast().map(Self::Block),
NodeKind::Unary => node.cast().map(Self::Unary),
NodeKind::Binary => node.cast().map(Self::Binary),
NodeKind::Call => node.cast().map(Self::Call),
NodeKind::Closure => node.cast().map(Self::Closure),
NodeKind::WithExpr => node.cast().map(Self::With),
NodeKind::LetExpr => node.cast().map(Self::Let),
NodeKind::IfExpr => node.cast().map(Self::If),
NodeKind::WhileExpr => node.cast().map(Self::While),
NodeKind::ForExpr => node.cast().map(Self::For),
NodeKind::ImportExpr => node.cast().map(Self::Import),
NodeKind::IncludeExpr => node.cast().map(Self::Include),
_ => node.cast().map(Self::Lit),
}
}
}
/// A literal: `1`, `true`, ... /// A literal: `1`, `true`, ...
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub enum Lit { pub enum Lit {
@ -335,17 +323,17 @@ pub enum Lit {
impl TypedNode for Lit { impl TypedNode for Lit {
fn from_red(node: RedRef) -> Option<Self> { fn from_red(node: RedRef) -> Option<Self> {
match node.kind() { match *node.kind() {
NodeKind::None => Some(Self::None(node.span())), NodeKind::None => Some(Self::None(node.span())),
NodeKind::Auto => Some(Self::Auto(node.span())), NodeKind::Auto => Some(Self::Auto(node.span())),
NodeKind::Bool(b) => Some(Self::Bool(node.span(), *b)), NodeKind::Bool(v) => Some(Self::Bool(node.span(), v)),
NodeKind::Int(i) => Some(Self::Int(node.span(), *i)), NodeKind::Int(v) => Some(Self::Int(node.span(), v)),
NodeKind::Float(f) => Some(Self::Float(node.span(), *f)), NodeKind::Float(v) => Some(Self::Float(node.span(), v)),
NodeKind::Length(f, unit) => Some(Self::Length(node.span(), *f, *unit)), NodeKind::Length(v, unit) => Some(Self::Length(node.span(), v, unit)),
NodeKind::Angle(f, unit) => Some(Self::Angle(node.span(), *f, *unit)), NodeKind::Angle(v, unit) => Some(Self::Angle(node.span(), v, unit)),
NodeKind::Percentage(f) => Some(Self::Percent(node.span(), *f)), NodeKind::Percentage(v) => Some(Self::Percent(node.span(), v)),
NodeKind::Fraction(f) => Some(Self::Fractional(node.span(), *f)), NodeKind::Fraction(v) => Some(Self::Fractional(node.span(), v)),
NodeKind::Str(s) => Some(Self::Str(node.span(), s.clone())), NodeKind::Str(ref v) => Some(Self::Str(node.span(), v.clone())),
_ => None, _ => None,
} }
} }
@ -354,17 +342,17 @@ impl TypedNode for Lit {
impl Lit { impl Lit {
/// The source code location. /// The source code location.
pub fn span(&self) -> Span { pub fn span(&self) -> Span {
match self { match *self {
Self::None(span) => *span, Self::None(span) => span,
Self::Auto(span) => *span, Self::Auto(span) => span,
Self::Bool(span, _) => *span, Self::Bool(span, _) => span,
Self::Int(span, _) => *span, Self::Int(span, _) => span,
Self::Float(span, _) => *span, Self::Float(span, _) => span,
Self::Length(span, _, _) => *span, Self::Length(span, _, _) => span,
Self::Angle(span, _, _) => *span, Self::Angle(span, _, _) => span,
Self::Percent(span, _) => *span, Self::Percent(span, _) => span,
Self::Fractional(span, _) => *span, Self::Fractional(span, _) => span,
Self::Str(span, _) => *span, Self::Str(span, _) => span,
} }
} }
} }
@ -401,16 +389,12 @@ node! {
impl Named { impl Named {
/// The name: `pattern`. /// The name: `pattern`.
pub fn name(&self) -> Ident { pub fn name(&self) -> Ident {
self.0.cast_first_child().expect("named pair is missing name ident") self.0.cast_first_child().expect("named pair is missing name")
} }
/// The right-hand side of the pair: `dashed`. /// The right-hand side of the pair: `dashed`.
pub fn expr(&self) -> Expr { pub fn expr(&self) -> Expr {
self.0 self.0.cast_last_child().expect("named pair is missing expression")
.children()
.filter_map(RedRef::cast)
.nth(1)
.expect("named pair is missing expression")
} }
} }
@ -422,9 +406,7 @@ node! {
impl TemplateExpr { impl TemplateExpr {
/// The contents of the template. /// The contents of the template.
pub fn body(&self) -> Markup { pub fn body(&self) -> Markup {
self.0 self.0.cast_first_child().expect("template is missing body")
.cast_first_child()
.expect("template expression is missing body")
} }
} }
@ -436,9 +418,7 @@ node! {
impl GroupExpr { impl GroupExpr {
/// The wrapped expression. /// The wrapped expression.
pub fn expr(&self) -> Expr { pub fn expr(&self) -> Expr {
self.0 self.0.cast_first_child().expect("group is missing expression")
.cast_first_child()
.expect("group expression is missing expression")
} }
} }
@ -469,9 +449,7 @@ impl UnaryExpr {
/// The expression to operator on: `x`. /// The expression to operator on: `x`.
pub fn expr(&self) -> Expr { pub fn expr(&self) -> Expr {
self.0 self.0.cast_last_child().expect("unary expression is missing child")
.cast_first_child()
.expect("unary expression is missing expression")
} }
} }
@ -506,7 +484,7 @@ impl UnOp {
/// The precedence of this operator. /// The precedence of this operator.
pub fn precedence(self) -> usize { pub fn precedence(self) -> usize {
match self { match self {
Self::Pos | Self::Neg => 8, Self::Pos | Self::Neg => 7,
Self::Not => 4, Self::Not => 4,
} }
} }
@ -544,9 +522,7 @@ impl BinaryExpr {
/// The right-hand side of the operation: `b`. /// The right-hand side of the operation: `b`.
pub fn rhs(&self) -> Expr { pub fn rhs(&self) -> Expr {
self.0 self.0
.children() .cast_last_child()
.filter_map(RedRef::cast)
.nth(1)
.expect("binary expression is missing right-hand side") .expect("binary expression is missing right-hand side")
} }
} }
@ -701,14 +677,12 @@ node! {
impl CallExpr { impl CallExpr {
/// The function to call. /// The function to call.
pub fn callee(&self) -> Expr { pub fn callee(&self) -> Expr {
self.0.cast_first_child().expect("call expression is missing callee") self.0.cast_first_child().expect("call is missing callee")
} }
/// The arguments to the function. /// The arguments to the function.
pub fn args(&self) -> CallArgs { pub fn args(&self) -> CallArgs {
self.0 self.0.cast_last_child().expect("call is missing argument list")
.cast_first_child()
.expect("call expression is missing argument list")
} }
} }
@ -738,14 +712,9 @@ pub enum CallArg {
impl TypedNode for CallArg { impl TypedNode for CallArg {
fn from_red(node: RedRef) -> Option<Self> { fn from_red(node: RedRef) -> Option<Self> {
match node.kind() { match node.kind() {
NodeKind::Named => Some(CallArg::Named( NodeKind::Named => node.cast().map(CallArg::Named),
node.cast().expect("named call argument is missing name"), NodeKind::Spread => node.cast_first_child().map(CallArg::Spread),
)), _ => node.cast().map(CallArg::Pos),
NodeKind::Spread => Some(CallArg::Spread(
node.cast_first_child()
.expect("call argument sink is missing expression"),
)),
_ => Some(CallArg::Pos(node.cast()?)),
} }
} }
} }
@ -754,8 +723,8 @@ impl CallArg {
/// The name of this argument. /// The name of this argument.
pub fn span(&self) -> Span { pub fn span(&self) -> Span {
match self { match self {
Self::Named(named) => named.span(),
Self::Pos(expr) => expr.span(), Self::Pos(expr) => expr.span(),
Self::Named(named) => named.span(),
Self::Spread(expr) => expr.span(), Self::Spread(expr) => expr.span(),
} }
} }
@ -771,8 +740,6 @@ impl ClosureExpr {
/// ///
/// This only exists if you use the function syntax sugar: `let f(x) = y`. /// This only exists if you use the function syntax sugar: `let f(x) = y`.
pub fn name(&self) -> Option<Ident> { pub fn name(&self) -> Option<Ident> {
// `first_convert_child` does not work here because of the Option in the
// Result.
self.0.cast_first_child() self.0.cast_first_child()
} }
@ -788,22 +755,11 @@ impl ClosureExpr {
/// The body of the closure. /// The body of the closure.
pub fn body(&self) -> Expr { pub fn body(&self) -> Expr {
// The filtering for the NodeKind is necessary here because otherwise,
// `first_convert_child` will use the Ident if present.
self.0.cast_last_child().expect("closure is missing body") self.0.cast_last_child().expect("closure is missing body")
} }
/// The red node reference of the body of the closure.
pub fn body_ref(&self) -> RedRef {
self.0
.children()
.filter(|x| x.cast::<Expr>().is_some())
.last()
.unwrap()
}
} }
/// An parameter to a closure. /// A parameter to a closure.
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub enum ClosureParam { pub enum ClosureParam {
/// A positional parameter: `x`. /// A positional parameter: `x`.
@ -817,17 +773,10 @@ pub enum ClosureParam {
impl TypedNode for ClosureParam { impl TypedNode for ClosureParam {
fn from_red(node: RedRef) -> Option<Self> { fn from_red(node: RedRef) -> Option<Self> {
match node.kind() { match node.kind() {
NodeKind::Ident(id) => { NodeKind::Ident(_) => node.cast().map(ClosureParam::Pos),
Some(ClosureParam::Pos(Ident::new_unchecked(id, node.span()))) NodeKind::Named => node.cast().map(ClosureParam::Named),
} NodeKind::Spread => node.cast_first_child().map(ClosureParam::Sink),
NodeKind::Named => Some(ClosureParam::Named( _ => None,
node.cast().expect("named closure parameter is missing name"),
)),
NodeKind::Spread => Some(ClosureParam::Sink(
node.cast_first_child()
.expect("closure parameter sink is missing identifier"),
)),
_ => Some(ClosureParam::Pos(node.cast()?)),
} }
} }
} }
@ -840,9 +789,7 @@ node! {
impl WithExpr { impl WithExpr {
/// The function to apply the arguments to. /// The function to apply the arguments to.
pub fn callee(&self) -> Expr { pub fn callee(&self) -> Expr {
self.0 self.0.cast_first_child().expect("with expression is missing callee")
.cast_first_child()
.expect("with expression is missing callee expression")
} }
/// The arguments to apply to the function. /// The arguments to apply to the function.
@ -861,17 +808,16 @@ node! {
impl LetExpr { impl LetExpr {
/// The binding to assign to. /// The binding to assign to.
pub fn binding(&self) -> Ident { pub fn binding(&self) -> Ident {
if let Some(c) = self.0.cast_first_child() { match self.0.cast_first_child() {
c Some(Expr::Ident(binding)) => binding,
} else if let Some(w) = self.0.typed_child(&NodeKind::WithExpr) { Some(Expr::With(with)) => match with.callee() {
// Can't do an `first_convert_child` here because the WithExpr's Expr::Ident(binding) => binding,
// callee has to be an identifier. _ => panic!("let .. with callee must be identifier"),
w.cast_first_child() },
.expect("with expression is missing an identifier callee") Some(Expr::Closure(closure)) => {
} else if let Some(Expr::Closure(c)) = self.0.cast_last_child() { closure.name().expect("let-bound closure is missing name")
c.name().expect("closure is missing an identifier name") }
} else { _ => panic!("let expression is missing binding"),
panic!("let expression is missing either an identifier or a with expression")
} }
} }
@ -880,24 +826,10 @@ impl LetExpr {
if self.0.cast_first_child::<Ident>().is_some() { if self.0.cast_first_child::<Ident>().is_some() {
self.0.children().filter_map(RedRef::cast).nth(1) self.0.children().filter_map(RedRef::cast).nth(1)
} else { } else {
Some( // This is a let .. with expression.
self.0 self.0.cast_first_child()
.cast_first_child()
.expect("let expression is missing a with expression"),
)
} }
} }
/// The red node reference for the expression the binding is initialized
/// with.
pub fn init_ref(&self) -> RedRef {
if self.0.cast_first_child::<Ident>().is_some() {
self.0.children().filter(|x| x.cast::<Expr>().is_some()).nth(1)
} else {
self.0.children().find(|x| x.cast::<Expr>().is_some())
}
.unwrap()
}
} }
node! { node! {
@ -908,16 +840,12 @@ node! {
impl ImportExpr { impl ImportExpr {
/// The items to be imported. /// The items to be imported.
pub fn imports(&self) -> Imports { pub fn imports(&self) -> Imports {
self.0 self.0.cast_first_child().expect("import is missing items")
.cast_first_child()
.expect("import expression is missing import list")
} }
/// The location of the importable file. /// The location of the importable file.
pub fn path(&self) -> Expr { pub fn path(&self) -> Expr {
self.0 self.0.cast_last_child().expect("import is missing path")
.cast_first_child()
.expect("import expression is missing path expression")
} }
} }
@ -926,8 +854,8 @@ impl ImportExpr {
pub enum Imports { pub enum Imports {
/// All items in the scope of the file should be imported. /// All items in the scope of the file should be imported.
Wildcard, Wildcard,
/// The specified identifiers from the file should be imported. /// The specified items from the file should be imported.
Idents(Vec<Ident>), Items(Vec<Ident>),
} }
impl TypedNode for Imports { impl TypedNode for Imports {
@ -935,8 +863,8 @@ impl TypedNode for Imports {
match node.kind() { match node.kind() {
NodeKind::Star => Some(Imports::Wildcard), NodeKind::Star => Some(Imports::Wildcard),
NodeKind::ImportItems => { NodeKind::ImportItems => {
let idents = node.children().filter_map(RedRef::cast).collect(); let items = node.children().filter_map(RedRef::cast).collect();
Some(Imports::Idents(idents)) Some(Imports::Items(items))
} }
_ => None, _ => None,
} }
@ -951,9 +879,7 @@ node! {
impl IncludeExpr { impl IncludeExpr {
/// The location of the file to be included. /// The location of the file to be included.
pub fn path(&self) -> Expr { pub fn path(&self) -> Expr {
self.0 self.0.cast_last_child().expect("include is missing path")
.cast_first_child()
.expect("include expression is missing path expression")
} }
} }
@ -965,9 +891,7 @@ node! {
impl IfExpr { impl IfExpr {
/// The condition which selects the body to evaluate. /// The condition which selects the body to evaluate.
pub fn condition(&self) -> Expr { pub fn condition(&self) -> Expr {
self.0 self.0.cast_first_child().expect("if expression is missing condition")
.cast_first_child()
.expect("if expression is missing condition expression")
} }
/// The expression to evaluate if the condition is true. /// The expression to evaluate if the condition is true.
@ -976,7 +900,7 @@ impl IfExpr {
.children() .children()
.filter_map(RedRef::cast) .filter_map(RedRef::cast)
.nth(1) .nth(1)
.expect("if expression is missing if body") .expect("if expression is missing body")
} }
/// The expression to evaluate if the condition is false. /// The expression to evaluate if the condition is false.
@ -993,18 +917,12 @@ node! {
impl WhileExpr { impl WhileExpr {
/// The condition which selects whether to evaluate the body. /// The condition which selects whether to evaluate the body.
pub fn condition(&self) -> Expr { pub fn condition(&self) -> Expr {
self.0 self.0.cast_first_child().expect("while loop is missing condition")
.cast_first_child()
.expect("while loop expression is missing condition expression")
} }
/// The expression to evaluate while the condition is true. /// The expression to evaluate while the condition is true.
pub fn body(&self) -> Expr { pub fn body(&self) -> Expr {
self.0 self.0.cast_last_child().expect("while loop is missing body")
.children()
.filter_map(RedRef::cast)
.nth(1)
.expect("while loop expression is missing body")
} }
} }
@ -1016,34 +934,17 @@ node! {
impl ForExpr { impl ForExpr {
/// The pattern to assign to. /// The pattern to assign to.
pub fn pattern(&self) -> ForPattern { pub fn pattern(&self) -> ForPattern {
self.0 self.0.cast_first_child().expect("for loop is missing pattern")
.cast_first_child()
.expect("for loop expression is missing pattern")
} }
/// The expression to iterate over. /// The expression to iterate over.
pub fn iter(&self) -> Expr { pub fn iter(&self) -> Expr {
self.0 self.0.cast_first_child().expect("for loop is missing iterable")
.cast_first_child()
.expect("for loop expression is missing iterable expression")
} }
/// The expression to evaluate for each iteration. /// The expression to evaluate for each iteration.
pub fn body(&self) -> Expr { pub fn body(&self) -> Expr {
self.0 self.0.cast_last_child().expect("for loop is missing body")
.children()
.filter_map(RedRef::cast)
.last()
.expect("for loop expression is missing body")
}
/// The red node reference for the expression to evaluate for each iteration.
pub fn body_ref(&self) -> RedRef {
self.0
.children()
.filter(|x| x.cast::<Expr>().is_some())
.last()
.unwrap()
} }
} }
@ -1062,19 +963,11 @@ impl ForPattern {
/// The value part of the pattern. /// The value part of the pattern.
pub fn value(&self) -> Ident { pub fn value(&self) -> Ident {
self.0 self.0.cast_last_child().expect("for loop pattern is missing value")
.cast_last_child()
.expect("for-in loop pattern is missing value")
} }
} }
/// An unicode identifier with a few extra permissible characters. /// An identifier.
///
/// In addition to what is specified in the [Unicode Standard][uax31], we allow:
/// - `_` as a starting character,
/// - `_` and `-` as continuing characters.
///
/// [uax31]: http://www.unicode.org/reports/tr31/
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub struct Ident { pub struct Ident {
/// The source code location. /// The source code location.
@ -1083,44 +976,13 @@ pub struct Ident {
pub string: EcoString, pub string: EcoString,
} }
impl Ident {
/// Create a new identifier from a string checking that it is a valid.
pub fn new(
string: impl AsRef<str> + Into<EcoString>,
span: impl Into<Span>,
) -> Option<Self> {
is_ident(string.as_ref())
.then(|| Self { span: span.into(), string: string.into() })
}
/// Create a new identifier from a string and a span.
///
/// The `string` must be a valid identifier.
#[track_caller]
pub fn new_unchecked(string: impl Into<EcoString>, span: Span) -> Self {
let string = string.into();
debug_assert!(is_ident(&string), "`{}` is not a valid identifier", string);
Self { span, string }
}
/// Return a reference to the underlying string.
pub fn as_str(&self) -> &str {
&self.string
}
}
impl Deref for Ident {
type Target = str;
fn deref(&self) -> &Self::Target {
self.as_str()
}
}
impl TypedNode for Ident { impl TypedNode for Ident {
fn from_red(node: RedRef) -> Option<Self> { fn from_red(node: RedRef) -> Option<Self> {
match node.kind() { match node.kind() {
NodeKind::Ident(string) => Some(Ident::new_unchecked(string, node.span())), NodeKind::Ident(string) => Some(Ident {
span: node.span(),
string: string.clone(),
}),
_ => None, _ => None,
} }
} }

View File

@ -5,7 +5,6 @@ mod pretty;
mod span; mod span;
use std::fmt::{self, Debug, Display, Formatter}; use std::fmt::{self, Debug, Display, Formatter};
use std::mem;
use std::rc::Rc; use std::rc::Rc;
pub use pretty::*; pub use pretty::*;
@ -40,14 +39,6 @@ impl Green {
self.data().kind() self.data().kind()
} }
/// Set the type of the node.
pub fn set_kind(&mut self, kind: NodeKind) {
match self {
Self::Node(node) => Rc::make_mut(node).data.set_kind(kind),
Self::Token(data) => data.set_kind(kind),
}
}
/// The length of the node. /// The length of the node.
pub fn len(&self) -> usize { pub fn len(&self) -> usize {
self.data().len() self.data().len()
@ -68,6 +59,18 @@ impl Green {
Green::Token(_) => &[], Green::Token(_) => &[],
} }
} }
/// Change the type of the node.
pub fn convert(&mut self, kind: NodeKind) {
match self {
Self::Node(node) => {
let node = Rc::make_mut(node);
node.erroneous |= kind.is_error();
node.data.kind = kind;
}
Self::Token(data) => data.kind = kind,
}
}
} }
impl Default for Green { impl Default for Green {
@ -161,11 +164,6 @@ impl GreenData {
&self.kind &self.kind
} }
/// Set the type of the node.
pub fn set_kind(&mut self, kind: NodeKind) {
self.kind = kind;
}
/// The length of the node. /// The length of the node.
pub fn len(&self) -> usize { pub fn len(&self) -> usize {
self.len self.len
@ -178,123 +176,7 @@ impl From<GreenData> for Green {
} }
} }
/// A borrowed wrapper for a [`GreenNode`] with span information. /// A owned wrapper for a green node with span information.
///
/// Borrowed variant of [`RedNode`]. Can be [cast](Self::cast) to an AST node.
#[derive(Copy, Clone, PartialEq)]
pub struct RedRef<'a> {
id: SourceId,
offset: usize,
green: &'a Green,
}
impl<'a> RedRef<'a> {
/// Convert to an owned representation.
pub fn own(self) -> RedNode {
RedNode {
id: self.id,
offset: self.offset,
green: self.green.clone(),
}
}
/// The type of the node.
pub fn kind(self) -> &'a NodeKind {
self.green.kind()
}
/// The length of the node.
pub fn len(self) -> usize {
self.green.len()
}
/// The span of the node.
pub fn span(self) -> Span {
Span::new(self.id, self.offset, self.offset + self.green.len())
}
/// Whether the node or its children contain an error.
pub fn erroneous(self) -> bool {
self.green.erroneous()
}
/// The error messages for this node and its descendants.
pub fn errors(self) -> Vec<Error> {
if !self.erroneous() {
return vec![];
}
match self.kind() {
NodeKind::Error(pos, msg) => {
let span = match pos {
ErrorPosition::Start => self.span().at_start(),
ErrorPosition::Full => self.span(),
ErrorPosition::End => self.span().at_end(),
};
vec![Error::new(span, msg.to_string())]
}
_ => self
.children()
.filter(|red| red.erroneous())
.flat_map(|red| red.errors())
.collect(),
}
}
/// Convert the node to a typed AST node.
pub fn cast<T>(self) -> Option<T>
where
T: TypedNode,
{
T::from_red(self)
}
/// The node's children.
pub fn children(self) -> impl Iterator<Item = RedRef<'a>> {
let children = match &self.green {
Green::Node(node) => node.children(),
Green::Token(_) => &[],
};
let mut cursor = self.offset;
children.iter().map(move |green| {
let offset = cursor;
cursor += green.len();
RedRef { id: self.id, offset, green }
})
}
/// Get the first child of some type.
pub(crate) fn typed_child(self, kind: &NodeKind) -> Option<RedRef<'a>> {
self.children()
.find(|x| mem::discriminant(x.kind()) == mem::discriminant(kind))
}
/// Get the first child that can cast to some AST type.
pub(crate) fn cast_first_child<T: TypedNode>(self) -> Option<T> {
self.children().find_map(RedRef::cast)
}
/// Get the last child that can cast to some AST type.
pub(crate) fn cast_last_child<T: TypedNode>(self) -> Option<T> {
self.children().filter_map(RedRef::cast).last()
}
}
impl Debug for RedRef<'_> {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
write!(f, "{:?}: {:?}", self.kind(), self.span())?;
let mut children = self.children().peekable();
if children.peek().is_some() {
f.write_str(" ")?;
f.debug_list().entries(children.map(RedRef::own)).finish()?;
}
Ok(())
}
}
/// A owned wrapper for a [`GreenNode`] with span information.
/// ///
/// Owned variant of [`RedRef`]. Can be [cast](Self::cast) to an AST nodes. /// Owned variant of [`RedRef`]. Can be [cast](Self::cast) to an AST nodes.
#[derive(Clone, PartialEq)] #[derive(Clone, PartialEq)]
@ -348,22 +230,17 @@ impl RedNode {
} }
/// The children of the node. /// The children of the node.
pub fn children(&self) -> impl Iterator<Item = RedRef<'_>> { pub fn children(&self) -> Children<'_> {
self.as_ref().children() self.as_ref().children()
} }
/// Get the first child of some type.
pub(crate) fn typed_child(&self, kind: &NodeKind) -> Option<RedNode> {
self.as_ref().typed_child(kind).map(RedRef::own)
}
/// Get the first child that can cast to some AST type. /// Get the first child that can cast to some AST type.
pub(crate) fn cast_first_child<T: TypedNode>(&self) -> Option<T> { pub fn cast_first_child<T: TypedNode>(&self) -> Option<T> {
self.as_ref().cast_first_child() self.as_ref().cast_first_child()
} }
/// Get the last child that can cast to some AST type. /// Get the last child that can cast to some AST type.
pub(crate) fn cast_last_child<T: TypedNode>(&self) -> Option<T> { pub fn cast_last_child<T: TypedNode>(&self) -> Option<T> {
self.as_ref().cast_last_child() self.as_ref().cast_last_child()
} }
} }
@ -374,6 +251,146 @@ impl Debug for RedNode {
} }
} }
/// A borrowed wrapper for a green node with span information.
///
/// Borrowed variant of [`RedNode`]. Can be [cast](Self::cast) to an AST node.
#[derive(Copy, Clone, PartialEq)]
pub struct RedRef<'a> {
id: SourceId,
offset: usize,
green: &'a Green,
}
impl<'a> RedRef<'a> {
/// Convert to an owned representation.
pub fn own(self) -> RedNode {
RedNode {
id: self.id,
offset: self.offset,
green: self.green.clone(),
}
}
/// The type of the node.
pub fn kind(self) -> &'a NodeKind {
self.green.kind()
}
/// The length of the node.
pub fn len(self) -> usize {
self.green.len()
}
/// The span of the node.
pub fn span(self) -> Span {
Span::new(self.id, self.offset, self.offset + self.green.len())
}
/// The error messages for this node and its descendants.
pub fn errors(self) -> Vec<Error> {
if !self.green.erroneous() {
return vec![];
}
match self.kind() {
NodeKind::Error(pos, msg) => {
let span = match pos {
ErrorPos::Start => self.span().at_start(),
ErrorPos::Full => self.span(),
ErrorPos::End => self.span().at_end(),
};
vec![Error::new(span, msg.to_string())]
}
_ => self
.children()
.filter(|red| red.green.erroneous())
.flat_map(|red| red.errors())
.collect(),
}
}
/// Convert the node to a typed AST node.
pub fn cast<T>(self) -> Option<T>
where
T: TypedNode,
{
T::from_red(self)
}
/// The node's children.
pub fn children(self) -> Children<'a> {
let children = match &self.green {
Green::Node(node) => node.children(),
Green::Token(_) => &[],
};
Children {
id: self.id,
iter: children.iter(),
front: self.offset,
back: self.offset + self.len(),
}
}
/// Get the first child that can cast to some AST type.
pub fn cast_first_child<T: TypedNode>(self) -> Option<T> {
self.children().find_map(RedRef::cast)
}
/// Get the last child that can cast to some AST type.
pub fn cast_last_child<T: TypedNode>(self) -> Option<T> {
self.children().rev().find_map(RedRef::cast)
}
}
impl Debug for RedRef<'_> {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
write!(f, "{:?}: {:?}", self.kind(), self.span())?;
let mut children = self.children().peekable();
if children.peek().is_some() {
f.write_str(" ")?;
f.debug_list().entries(children.map(RedRef::own)).finish()?;
}
Ok(())
}
}
/// An iterator over the children of a red node.
pub struct Children<'a> {
id: SourceId,
iter: std::slice::Iter<'a, Green>,
front: usize,
back: usize,
}
impl<'a> Iterator for Children<'a> {
type Item = RedRef<'a>;
fn next(&mut self) -> Option<Self::Item> {
self.iter.next().map(|green| {
let offset = self.front;
self.front += green.len();
RedRef { id: self.id, offset, green }
})
}
fn size_hint(&self) -> (usize, Option<usize>) {
self.iter.size_hint()
}
}
impl DoubleEndedIterator for Children<'_> {
fn next_back(&mut self) -> Option<Self::Item> {
self.iter.next_back().map(|green| {
self.back -= green.len();
RedRef { id: self.id, offset: self.back, green }
})
}
}
impl ExactSizeIterator for Children<'_> {}
/// All syntactical building blocks that can be part of a Typst document. /// All syntactical building blocks that can be part of a Typst document.
/// ///
/// Can be emitted as a token by the tokenizer or as part of a green node by /// Can be emitted as a token by the tokenizer or as part of a green node by
@ -533,7 +550,7 @@ pub enum NodeKind {
Array, Array,
/// A dictionary expression: `(thickness: 3pt, pattern: dashed)`. /// A dictionary expression: `(thickness: 3pt, pattern: dashed)`.
Dict, Dict,
/// A named argument: `thickness: 3pt`. /// A named pair: `thickness: 3pt`.
Named, Named,
/// A grouped expression: `(1 + 2)`. /// A grouped expression: `(1 + 2)`.
Group, Group,
@ -582,12 +599,12 @@ pub enum NodeKind {
/// The comment can contain nested block comments. /// The comment can contain nested block comments.
BlockComment, BlockComment,
/// Tokens that appear in the wrong place. /// Tokens that appear in the wrong place.
Error(ErrorPosition, EcoString), Error(ErrorPos, EcoString),
/// Unknown character sequences. /// Unknown character sequences.
Unknown(EcoString), Unknown(EcoString),
} }
/// Payload of a raw block: `` `...` ``. /// Payload of a raw block node.
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub struct RawData { pub struct RawData {
/// The raw text in the block. /// The raw text in the block.
@ -600,19 +617,19 @@ pub struct RawData {
pub block: bool, pub block: bool,
} }
/// Payload of a math formula: `$2pi + x$` or `$[f'(x) = x^2]$`. /// Payload of a math formula node.
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub struct MathData { pub struct MathData {
/// The formula between the dollars. /// The formula between the dollars / brackets.
pub formula: EcoString, pub formula: EcoString,
/// Whether the formula is display-level, that is, it is surrounded by /// Whether the formula is display-level, that is, it is surrounded by
/// `$[..]`. /// `$[..]$`.
pub display: bool, pub display: bool,
} }
/// Where in a node an error should be annotated. /// Where in a node an error should be annotated.
#[derive(Debug, Copy, Clone, PartialEq, Eq)] #[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub enum ErrorPosition { pub enum ErrorPos {
/// At the start of the node. /// At the start of the node.
Start, Start,
/// Over the full width of the node. /// Over the full width of the node.

View File

@ -141,7 +141,7 @@ impl Pretty for RawNode {
// Language tag. // Language tag.
if let Some(lang) = &self.lang { if let Some(lang) = &self.lang {
lang.pretty(p); p.push_str(lang);
} }
// Start untrimming. // Start untrimming.
@ -492,7 +492,7 @@ impl Pretty for Imports {
fn pretty(&self, p: &mut Printer) { fn pretty(&self, p: &mut Printer) {
match self { match self {
Self::Wildcard => p.push('*'), Self::Wildcard => p.push('*'),
Self::Idents(idents) => { Self::Items(idents) => {
p.join(idents, ", ", |item, p| item.pretty(p)); p.join(idents, ", ", |item, p| item.pretty(p));
} }
} }
@ -508,7 +508,7 @@ impl Pretty for IncludeExpr {
impl Pretty for Ident { impl Pretty for Ident {
fn pretty(&self, p: &mut Printer) { fn pretty(&self, p: &mut Printer) {
p.push_str(self.as_str()); p.push_str(&self.string);
} }
} }