caisin/typst
1
0
Fork 0
mirror of https://github.com/typst/typst synced 2025-05-15 01:25:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactoring

Browse Source 
Co-Authored-By: Martin <mhaug@live.de>
This commit is contained in:
Laurenz 2021-11-02 12:13:45 +01:00 committed by Martin Haug
parent 42afb27cef
commit 65fac0e57c
17 changed files with 338 additions and 427 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
src/diag.rs
View File

@ -100,7 +100,7 @@ impl<T> Trace<T> for TypResult<T> {
{ {
self.map_err(|mut errors| { self.map_err(|mut errors| {
for error in errors.iter_mut() { for error in errors.iter_mut() {
if !span.contains(error.span) { if !span.surrounds(error.span) {
error.trace.push(Spanned::new(make_point(), span)); error.trace.push(Spanned::new(make_point(), span));
} }
} }

3
src/eval/capture.rs
View File

@ -1,7 +1,8 @@
use std::rc::Rc; use std::rc::Rc;
use super::{Scope, Scopes, Value}; use super::{Scope, Scopes, Value};
use crate::syntax::{ClosureParam, Expr, Imports, RedRef}; use crate::syntax::ast::{ClosureParam, Expr, Imports};
use crate::syntax::RedRef;
/// A visitor that captures variable slots. /// A visitor that captures variable slots.
pub struct CapturesVisitor<'a> { pub struct CapturesVisitor<'a> {

5
src/eval/mod.rs
View File

@ -36,7 +36,8 @@ use crate::geom::{Angle, Fractional, Length, Relative};
use crate::image::ImageStore; use crate::image::ImageStore;
use crate::loading::Loader; use crate::loading::Loader;
use crate::source::{SourceId, SourceStore}; use crate::source::{SourceId, SourceStore};
use crate::syntax::*; use crate::syntax::ast::*;
use crate::syntax::{Span, Spanned};
use crate::util::RefMutExt; use crate::util::RefMutExt;
use crate::Context; use crate::Context;
@ -238,7 +239,7 @@ impl Eval for DictExpr {
fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> { fn eval(&self, ctx: &mut EvalContext) -> TypResult<Self::Output> {
self.items() self.items()
.map(|x| Ok(((&x.name().string).into(), x.expr().eval(ctx)?))) .map(|x| Ok((x.name().string.into(), x.expr().eval(ctx)?)))
.collect() .collect()
} }
} }

2
src/eval/walk.rs
View File

@ -5,7 +5,7 @@ use crate::diag::TypResult;
use crate::geom::Spec; use crate::geom::Spec;
use crate::layout::BlockLevel; use crate::layout::BlockLevel;
use crate::library::{GridNode, ParChild, ParNode, TrackSizing}; use crate::library::{GridNode, ParChild, ParNode, TrackSizing};
use crate::syntax::*; use crate::syntax::ast::*;
use crate::util::BoolExt; use crate::util::BoolExt;
/// Walk markup, filling the currently built template. /// Walk markup, filling the currently built template.

2
src/geom/relative.rs
View File

@ -3,7 +3,7 @@ use super::*;
/// A relative length. /// A relative length.
/// ///
/// _Note_: `50%` is represented as `0.5` here, but stored as `50.0` in the /// _Note_: `50%` is represented as `0.5` here, but stored as `50.0` in the
/// corresponding [literal](crate::syntax::Lit::Percent). /// corresponding [literal](crate::syntax::ast::Lit::Percent).
#[derive(Default, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] #[derive(Default, Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
pub struct Relative(N64); pub struct Relative(N64);

2
src/lib.rs
View File

@ -20,7 +20,7 @@
//! //!
//! [tokens]: parse::Tokens //! [tokens]: parse::Tokens
//! [parsed]: parse::parse //! [parsed]: parse::parse
//! [markup]: syntax::Markup //! [markup]: syntax::ast::Markup
//! [evaluate]: eval::eval //! [evaluate]: eval::eval
//! [module]: eval::Module //! [module]: eval::Module
//! [layout tree]: layout::LayoutTree //! [layout tree]: layout::LayoutTree

3
src/parse/mod.rs
View File

@ -12,7 +12,8 @@ pub use tokens::*;
use std::rc::Rc; use std::rc::Rc;
use crate::syntax::*; use crate::syntax::ast::{Associativity, BinOp, UnOp};
use crate::syntax::{ErrorPosition, GreenNode, NodeKind};
use crate::util::EcoString; use crate::util::EcoString;
/// Parse a source file. /// Parse a source file.

8
src/parse/parser.rs
View File

@ -1,7 +1,7 @@
use std::ops::Range; use std::ops::Range;
use std::rc::Rc; use std::rc::Rc;
use super::{is_newline, TokenMode, Tokens}; use super::{TokenMode, Tokens};
use crate::syntax::{ErrorPosition, Green, GreenData, GreenNode, NodeKind}; use crate::syntax::{ErrorPosition, Green, GreenData, GreenNode, NodeKind};
use crate::util::EcoString; use crate::util::EcoString;
@ -375,11 +375,7 @@ impl<'s> Parser<'s> {
/// Determine the column index for the given byte index. /// Determine the column index for the given byte index.
pub fn column(&self, index: usize) -> usize { pub fn column(&self, index: usize) -> usize {
self.src[.. index] self.tokens.column(index)
.chars()
.rev()
.take_while(|&c| !is_newline(c))
.count()
} }
/// Slice out part of the source string. /// Slice out part of the source string.

49
src/parse/scanner.rs
View File

@ -1,5 +1,7 @@
use std::slice::SliceIndex; use std::slice::SliceIndex;
use unicode_xid::UnicodeXID;
/// A featureful char-based scanner. /// A featureful char-based scanner.
#[derive(Copy, Clone)] #[derive(Copy, Clone)]
pub struct Scanner<'s> { pub struct Scanner<'s> {
@ -106,16 +108,6 @@ impl<'s> Scanner<'s> {
self.index self.index
} }
/// The column index of a given index in the source string.
#[inline]
pub fn column(&self, index: usize) -> usize {
self.src[.. index]
.chars()
.rev()
.take_while(|&c| !is_newline(c))
.count()
}
/// Jump to an index in the source string. /// Jump to an index in the source string.
#[inline] #[inline]
pub fn jump(&mut self, index: usize) { pub fn jump(&mut self, index: usize) {
@ -124,6 +116,12 @@ impl<'s> Scanner<'s> {
self.index = index; self.index = index;
} }
/// The full source string.
#[inline]
pub fn src(&self) -> &'s str {
&self.src
}
/// Slice out part of the source string. /// Slice out part of the source string.
#[inline] #[inline]
pub fn get<I>(&self, index: I) -> &'s str pub fn get<I>(&self, index: I) -> &'s str
@ -160,6 +158,16 @@ impl<'s> Scanner<'s> {
// optimized away in some cases. // optimized away in some cases.
self.src.get(start .. self.index).unwrap_or_default() self.src.get(start .. self.index).unwrap_or_default()
} }
/// The column index of a given index in the source string.
#[inline]
pub fn column(&self, index: usize) -> usize {
self.src[.. index]
.chars()
.rev()
.take_while(|&c| !is_newline(c))
.count()
}
} }
/// Whether this character denotes a newline. /// Whether this character denotes a newline.
@ -173,3 +181,24 @@ pub fn is_newline(character: char) -> bool {
'\u{0085}' | '\u{2028}' | '\u{2029}' '\u{0085}' | '\u{2028}' | '\u{2029}'
) )
} }
/// Whether a string is a valid identifier.
#[inline]
pub fn is_ident(string: &str) -> bool {
let mut chars = string.chars();
chars
.next()
.map_or(false, |c| is_id_start(c) && chars.all(is_id_continue))
}
/// Whether a character can start an identifier.
#[inline]
pub fn is_id_start(c: char) -> bool {
c.is_xid_start() || c == '_'
}
/// Whether a character can continue an identifier.
#[inline]
pub fn is_id_continue(c: char) -> bool {
c.is_xid_continue() || c == '_' || c == '-'
}

108
src/parse/tokens.rs
View File

@ -1,11 +1,13 @@
use super::{is_newline, resolve_raw, Scanner}; use std::rc::Rc;
use super::{
is_id_continue, is_id_start, is_newline, resolve_hex, resolve_raw, resolve_string,
Scanner,
};
use crate::geom::{AngularUnit, LengthUnit}; use crate::geom::{AngularUnit, LengthUnit};
use crate::parse::resolve::{resolve_hex, resolve_string};
use crate::syntax::*; use crate::syntax::*;
use crate::util::EcoString; use crate::util::EcoString;
use std::rc::Rc;
/// An iterator over the tokens of a string of source code. /// An iterator over the tokens of a string of source code.
pub struct Tokens<'s> { pub struct Tokens<'s> {
s: Scanner<'s>, s: Scanner<'s>,
@ -55,6 +57,12 @@ impl<'s> Tokens<'s> {
self.s.jump(index); self.s.jump(index);
} }
/// The column of a given index in the source string.
#[inline]
pub fn column(&self, index: usize) -> usize {
self.s.column(index)
}
/// The underlying scanner. /// The underlying scanner.
#[inline] #[inline]
pub fn scanner(&self) -> Scanner<'s> { pub fn scanner(&self) -> Scanner<'s> {
@ -237,10 +245,8 @@ impl<'s> Tokens<'s> {
let sequence: EcoString = self.s.eat_while(|c| c.is_ascii_alphanumeric()).into(); let sequence: EcoString = self.s.eat_while(|c| c.is_ascii_alphanumeric()).into();
if self.s.eat_if('}') { if self.s.eat_if('}') {
if let Some(character) = resolve_hex(&sequence) { if let Some(c) = resolve_hex(&sequence) {
NodeKind::UnicodeEscape(UnicodeEscapeData { NodeKind::UnicodeEscape(c)
character,
})
} else { } else {
NodeKind::Error( NodeKind::Error(
ErrorPosition::Full, ErrorPosition::Full,
@ -308,7 +314,8 @@ impl<'s> Tokens<'s> {
} }
fn raw(&mut self) -> NodeKind { fn raw(&mut self) -> NodeKind {
let column = self.s.column(self.s.index() - 1); let column = self.column(self.s.index() - 1);
let mut backticks = 1; let mut backticks = 1;
while self.s.eat_if('`') && backticks < u8::MAX { while self.s.eat_if('`') && backticks < u8::MAX {
backticks += 1; backticks += 1;
@ -486,7 +493,7 @@ impl<'s> Tokens<'s> {
} }
})); }));
if self.s.eat_if('"') { if self.s.eat_if('"') {
NodeKind::Str(StrData { string }) NodeKind::Str(string)
} else { } else {
NodeKind::Error(ErrorPosition::End, "expected quote".into()) NodeKind::Error(ErrorPosition::End, "expected quote".into())
} }
@ -556,12 +563,13 @@ mod tests {
use super::*; use super::*;
use ErrorPosition::*;
use NodeKind::*; use NodeKind::*;
use Option::None; use Option::None;
use TokenMode::{Code, Markup}; use TokenMode::{Code, Markup};
fn UnicodeEscape(character: char) -> NodeKind { fn UnicodeEscape(c: char) -> NodeKind {
NodeKind::UnicodeEscape(UnicodeEscapeData { character }) NodeKind::UnicodeEscape(c)
} }
fn Error(pos: ErrorPosition, message: &str) -> NodeKind { fn Error(pos: ErrorPosition, message: &str) -> NodeKind {
@ -577,24 +585,12 @@ mod tests {
})) }))
} }
fn Math(formula: &str, display: bool, err_msg: Option<&str>) -> NodeKind { fn Math(formula: &str, display: bool) -> NodeKind {
match err_msg { NodeKind::Math(Rc::new(MathData { formula: formula.into(), display }))
None => {
NodeKind::Math(Rc::new(MathData { formula: formula.into(), display }))
}
Some(msg) => NodeKind::Error(
ErrorPosition::End,
format!("expected closing {}", msg).into(),
),
}
} }
fn Str(string: &str, terminated: bool) -> NodeKind { fn Str(string: &str) -> NodeKind {
if terminated { NodeKind::Str(string.into())
NodeKind::Str(StrData { string: string.into() })
} else {
NodeKind::Error(ErrorPosition::End, "expected quote".into())
}
} }
fn Text(string: &str) -> NodeKind { fn Text(string: &str) -> NodeKind {
@ -659,7 +655,7 @@ mod tests {
('/', None, "//", LineComment), ('/', None, "//", LineComment),
('/', None, "/**/", BlockComment), ('/', None, "/**/", BlockComment),
('/', Some(Markup), "*", Strong), ('/', Some(Markup), "*", Strong),
('/', Some(Markup), "$ $", Math(" ", false, None)), ('/', Some(Markup), "$ $", Math(" ", false)),
('/', Some(Markup), r"\\", Text("\\")), ('/', Some(Markup), r"\\", Text("\\")),
('/', Some(Markup), "#let", Let), ('/', Some(Markup), "#let", Let),
('/', Some(Code), "(", LeftParen), ('/', Some(Code), "(", LeftParen),
@ -781,16 +777,16 @@ mod tests {
t!(Markup[" /"]: r#"\""# => Text(r"\"), Text("\"")); t!(Markup[" /"]: r#"\""# => Text(r"\"), Text("\""));
// Test basic unicode escapes. // Test basic unicode escapes.
t!(Markup: r"\u{}" => Error(ErrorPosition::Full, "invalid unicode escape sequence")); t!(Markup: r"\u{}" => Error(Full, "invalid unicode escape sequence"));
t!(Markup: r"\u{2603}" => UnicodeEscape('☃')); t!(Markup: r"\u{2603}" => UnicodeEscape('☃'));
t!(Markup: r"\u{P}" => Error(ErrorPosition::Full, "invalid unicode escape sequence")); t!(Markup: r"\u{P}" => Error(Full, "invalid unicode escape sequence"));
// Test unclosed unicode escapes. // Test unclosed unicode escapes.
t!(Markup[" /"]: r"\u{" => Error(ErrorPosition::End, "expected closing brace")); t!(Markup[" /"]: r"\u{" => Error(End, "expected closing brace"));
t!(Markup[" /"]: r"\u{1" => Error(ErrorPosition::End, "expected closing brace")); t!(Markup[" /"]: r"\u{1" => Error(End, "expected closing brace"));
t!(Markup[" /"]: r"\u{26A4" => Error(ErrorPosition::End, "expected closing brace")); t!(Markup[" /"]: r"\u{26A4" => Error(End, "expected closing brace"));
t!(Markup[" /"]: r"\u{1Q3P" => Error(ErrorPosition::End, "expected closing brace")); t!(Markup[" /"]: r"\u{1Q3P" => Error(End, "expected closing brace"));
t!(Markup: r"\u{1🏕}" => Error(ErrorPosition::End, "expected closing brace"), Text("🏕"), RightBrace); t!(Markup: r"\u{1🏕}" => Error(End, "expected closing brace"), Text("🏕"), RightBrace);
} }
#[test] #[test]
@ -882,11 +878,11 @@ mod tests {
// Test basic raw block. // Test basic raw block.
t!(Markup: "``" => Raw("", None, 1, false)); t!(Markup: "``" => Raw("", None, 1, false));
t!(Markup: "`raw`" => Raw("raw", None, 1, false)); t!(Markup: "`raw`" => Raw("raw", None, 1, false));
t!(Markup[""]: "`]" => Error(ErrorPosition::End, "expected 1 backtick")); t!(Markup[""]: "`]" => Error(End, "expected 1 backtick"));
// Test special symbols in raw block. // Test special symbols in raw block.
t!(Markup: "`[brackets]`" => Raw("[brackets]", None, 1, false)); t!(Markup: "`[brackets]`" => Raw("[brackets]", None, 1, false));
t!(Markup[""]: r"`\`` " => Raw(r"\", None, 1, false), Error(ErrorPosition::End, "expected 1 backtick")); t!(Markup[""]: r"`\`` " => Raw(r"\", None, 1, false), Error(End, "expected 1 backtick"));
// Test separated closing backticks. // Test separated closing backticks.
t!(Markup: "```not `y`e`t```" => Raw("`y`e`t", Some("not"), 3, false)); t!(Markup: "```not `y`e`t```" => Raw("`y`e`t", Some("not"), 3, false));
@ -894,28 +890,28 @@ mod tests {
// Test more backticks. // Test more backticks.
t!(Markup: "``nope``" => Raw("", None, 1, false), Text("nope"), Raw("", None, 1, false)); t!(Markup: "``nope``" => Raw("", None, 1, false), Text("nope"), Raw("", None, 1, false));
t!(Markup: "````🚀````" => Raw("", Some("🚀"), 4, false)); t!(Markup: "````🚀````" => Raw("", Some("🚀"), 4, false));
t!(Markup[""]: "`````👩‍🚀````noend" => Error(ErrorPosition::End, "expected 5 backticks")); t!(Markup[""]: "`````👩‍🚀````noend" => Error(End, "expected 5 backticks"));
t!(Markup[""]: "````raw``````" => Raw("", Some("raw"), 4, false), Raw("", None, 1, false)); t!(Markup[""]: "````raw``````" => Raw("", Some("raw"), 4, false), Raw("", None, 1, false));
} }
#[test] #[test]
fn test_tokenize_math_formulas() { fn test_tokenize_math_formulas() {
// Test basic formula. // Test basic formula.
t!(Markup: "$$" => Math("", false, None)); t!(Markup: "$$" => Math("", false));
t!(Markup: "$x$" => Math("x", false, None)); t!(Markup: "$x$" => Math("x", false));
t!(Markup: r"$\\$" => Math(r"\\", false, None)); t!(Markup: r"$\\$" => Math(r"\\", false));
t!(Markup: "$[x + y]$" => Math("x + y", true, None)); t!(Markup: "$[x + y]$" => Math("x + y", true));
t!(Markup: r"$[\\]$" => Math(r"\\", true, None)); t!(Markup: r"$[\\]$" => Math(r"\\", true));
// Test unterminated. // Test unterminated.
t!(Markup[""]: "$x" => Math("x", false, Some("dollar sign"))); t!(Markup[""]: "$x" => Error(End, "expected closing dollar sign"));
t!(Markup[""]: "$[x" => Math("x", true, Some("bracket and dollar sign"))); t!(Markup[""]: "$[x" => Error(End, "expected closing bracket and dollar sign"));
t!(Markup[""]: "$[x]\n$" => Math("x]\n$", true, Some("bracket and dollar sign"))); t!(Markup[""]: "$[x]\n$" => Error(End, "expected closing bracket and dollar sign"));
// Test escape sequences. // Test escape sequences.
t!(Markup: r"$\$x$" => Math(r"\$x", false, None)); t!(Markup: r"$\$x$" => Math(r"\$x", false));
t!(Markup: r"$[\\\]$]$" => Math(r"\\\]$", true, None)); t!(Markup: r"$[\\\]$]$" => Math(r"\\\]$", true));
t!(Markup[""]: r"$[ ]\\$" => Math(r" ]\\$", true, Some("bracket and dollar sign"))); t!(Markup[""]: r"$[ ]\\$" => Error(End, "expected closing bracket and dollar sign"));
} }
#[test] #[test]
@ -1003,16 +999,16 @@ mod tests {
#[test] #[test]
fn test_tokenize_strings() { fn test_tokenize_strings() {
// Test basic strings. // Test basic strings.
t!(Code: "\"hi\"" => Str("hi", true)); t!(Code: "\"hi\"" => Str("hi"));
t!(Code: "\"hi\nthere\"" => Str("hi\nthere", true)); t!(Code: "\"hi\nthere\"" => Str("hi\nthere"));
t!(Code: "\"🌎\"" => Str("🌎", true)); t!(Code: "\"🌎\"" => Str("🌎"));
// Test unterminated. // Test unterminated.
t!(Code[""]: "\"hi" => Str("hi", false)); t!(Code[""]: "\"hi" => Error(End, "expected quote"));
// Test escaped quote. // Test escaped quote.
t!(Code: r#""a\"bc""# => Str("a\"bc", true)); t!(Code: r#""a\"bc""# => Str("a\"bc"));
t!(Code[""]: r#""\""# => Str("\"", false)); t!(Code[""]: r#""\""# => Error(End, "expected quote"));
} }
#[test] #[test]

3
src/source.rs
View File

@ -11,7 +11,8 @@ use serde::{Deserialize, Serialize};
use crate::diag::TypResult; use crate::diag::TypResult;
use crate::loading::{FileHash, Loader}; use crate::loading::{FileHash, Loader};
use crate::parse::{is_newline, parse, Scanner}; use crate::parse::{is_newline, parse, Scanner};
use crate::syntax::{GreenNode, Markup, RedNode}; use crate::syntax::ast::Markup;
use crate::syntax::{GreenNode, RedNode};
use crate::util::PathExt; use crate::util::PathExt;
#[cfg(feature = "codespan-reporting")] #[cfg(feature = "codespan-reporting")]

141
src/syntax/ast.rs
View File

@ -1,7 +1,18 @@
use super::{Ident, NodeKind, RedNode, RedRef, Span, TypedNode}; //! A typed layer over the red-green tree.
use std::ops::Deref;
use super::{NodeKind, RedNode, RedRef, Span};
use crate::geom::{AngularUnit, LengthUnit}; use crate::geom::{AngularUnit, LengthUnit};
use crate::parse::is_ident;
use crate::util::EcoString; use crate::util::EcoString;
/// A typed AST node.
pub trait TypedNode: Sized {
/// Convert from a red node to a typed node.
fn from_red(value: RedRef) -> Option<Self>;
}
macro_rules! node { macro_rules! node {
($(#[$attr:meta])* $name:ident) => { ($(#[$attr:meta])* $name:ident) => {
node!{$(#[$attr])* $name => $name} node!{$(#[$attr])* $name => $name}
@ -13,7 +24,7 @@ macro_rules! node {
pub struct $name(RedNode); pub struct $name(RedNode);
impl TypedNode for $name { impl TypedNode for $name {
fn cast_from(node: RedRef) -> Option<Self> { fn from_red(node: RedRef) -> Option<Self> {
if node.kind() != &NodeKind::$variant { if node.kind() != &NodeKind::$variant {
return None; return None;
} }
@ -23,10 +34,12 @@ macro_rules! node {
} }
impl $name { impl $name {
/// The source code location.
pub fn span(&self) -> Span { pub fn span(&self) -> Span {
self.0.span() self.0.span()
} }
/// The underlying red node.
pub fn underlying(&self) -> RedRef { pub fn underlying(&self) -> RedRef {
self.0.as_ref() self.0.as_ref()
} }
@ -40,7 +53,8 @@ node! {
} }
impl Markup { impl Markup {
pub fn nodes<'a>(&'a self) -> impl Iterator<Item = MarkupNode> + 'a { /// The markup nodes.
pub fn nodes(&self) -> impl Iterator<Item = MarkupNode> + '_ {
self.0.children().filter_map(RedRef::cast) self.0.children().filter_map(RedRef::cast)
} }
} }
@ -73,7 +87,7 @@ pub enum MarkupNode {
} }
impl TypedNode for MarkupNode { impl TypedNode for MarkupNode {
fn cast_from(node: RedRef) -> Option<Self> { fn from_red(node: RedRef) -> Option<Self> {
match node.kind() { match node.kind() {
NodeKind::Space(_) => Some(MarkupNode::Space), NodeKind::Space(_) => Some(MarkupNode::Space),
NodeKind::Linebreak => Some(MarkupNode::Linebreak), NodeKind::Linebreak => Some(MarkupNode::Linebreak),
@ -81,17 +95,14 @@ impl TypedNode for MarkupNode {
NodeKind::Strong => Some(MarkupNode::Strong), NodeKind::Strong => Some(MarkupNode::Strong),
NodeKind::Emph => Some(MarkupNode::Emph), NodeKind::Emph => Some(MarkupNode::Emph),
NodeKind::Text(s) => Some(MarkupNode::Text(s.clone())), NodeKind::Text(s) => Some(MarkupNode::Text(s.clone())),
NodeKind::UnicodeEscape(u) => Some(MarkupNode::Text(u.character.into())), NodeKind::UnicodeEscape(c) => Some(MarkupNode::Text((*c).into())),
NodeKind::EnDash => Some(MarkupNode::Text(EcoString::from("\u{2013}"))), NodeKind::EnDash => Some(MarkupNode::Text("\u{2013}".into())),
NodeKind::EmDash => Some(MarkupNode::Text(EcoString::from("\u{2014}"))), NodeKind::EmDash => Some(MarkupNode::Text("\u{2014}".into())),
NodeKind::NonBreakingSpace => { NodeKind::NonBreakingSpace => Some(MarkupNode::Text("\u{00A0}".into())),
Some(MarkupNode::Text(EcoString::from("\u{00A0}")))
}
NodeKind::Raw(_) => node.cast().map(MarkupNode::Raw), NodeKind::Raw(_) => node.cast().map(MarkupNode::Raw),
NodeKind::Heading => node.cast().map(MarkupNode::Heading), NodeKind::Heading => node.cast().map(MarkupNode::Heading),
NodeKind::List => node.cast().map(MarkupNode::List), NodeKind::List => node.cast().map(MarkupNode::List),
NodeKind::Enum => node.cast().map(MarkupNode::Enum), NodeKind::Enum => node.cast().map(MarkupNode::Enum),
NodeKind::Error(_, _) => None,
_ => node.cast().map(MarkupNode::Expr), _ => node.cast().map(MarkupNode::Expr),
} }
} }
@ -111,16 +122,16 @@ pub struct RawNode {
} }
impl TypedNode for RawNode { impl TypedNode for RawNode {
fn cast_from(node: RedRef) -> Option<Self> { fn from_red(node: RedRef) -> Option<Self> {
match node.kind() { match node.kind() {
NodeKind::Raw(raw) => { NodeKind::Raw(raw) => {
let span = node.span(); let full = node.span();
let start = span.start + raw.backticks as usize; let start = full.start + raw.backticks as usize;
Some(Self { Some(Self {
block: raw.block, block: raw.block,
lang: raw.lang.as_ref().and_then(|x| { lang: raw.lang.as_ref().and_then(|lang| {
let span = Span::new(span.source, start, start + x.len()); let span = Span::new(full.source, start, start + lang.len());
Ident::new(x, span) Ident::new(lang, span)
}), }),
text: raw.text.clone(), text: raw.text.clone(),
}) })
@ -272,7 +283,7 @@ impl Expr {
} }
impl TypedNode for Expr { impl TypedNode for Expr {
fn cast_from(node: RedRef) -> Option<Self> { fn from_red(node: RedRef) -> Option<Self> {
match node.kind() { match node.kind() {
NodeKind::Ident(_) => node.cast().map(Self::Ident), NodeKind::Ident(_) => node.cast().map(Self::Ident),
NodeKind::Array => node.cast().map(Self::Array), NodeKind::Array => node.cast().map(Self::Array),
@ -325,7 +336,7 @@ pub enum Lit {
} }
impl TypedNode for Lit { impl TypedNode for Lit {
fn cast_from(node: RedRef) -> Option<Self> { fn from_red(node: RedRef) -> Option<Self> {
match node.kind() { match node.kind() {
NodeKind::None => Some(Self::None(node.span())), NodeKind::None => Some(Self::None(node.span())),
NodeKind::Auto => Some(Self::Auto(node.span())), NodeKind::Auto => Some(Self::Auto(node.span())),
@ -336,13 +347,14 @@ impl TypedNode for Lit {
NodeKind::Angle(f, unit) => Some(Self::Angle(node.span(), *f, *unit)), NodeKind::Angle(f, unit) => Some(Self::Angle(node.span(), *f, *unit)),
NodeKind::Percentage(f) => Some(Self::Percent(node.span(), *f)), NodeKind::Percentage(f) => Some(Self::Percent(node.span(), *f)),
NodeKind::Fraction(f) => Some(Self::Fractional(node.span(), *f)), NodeKind::Fraction(f) => Some(Self::Fractional(node.span(), *f)),
NodeKind::Str(s) => Some(Self::Str(node.span(), s.string.clone())), NodeKind::Str(s) => Some(Self::Str(node.span(), s.clone())),
_ => None, _ => None,
} }
} }
} }
impl Lit { impl Lit {
/// The source code location.
pub fn span(&self) -> Span { pub fn span(&self) -> Span {
match self { match self {
Self::None(span) => *span, Self::None(span) => *span,
@ -366,7 +378,7 @@ node! {
impl ArrayExpr { impl ArrayExpr {
/// The array items. /// The array items.
pub fn items<'a>(&'a self) -> impl Iterator<Item = Expr> + 'a { pub fn items(&self) -> impl Iterator<Item = Expr> + '_ {
self.0.children().filter_map(RedRef::cast) self.0.children().filter_map(RedRef::cast)
} }
} }
@ -378,7 +390,7 @@ node! {
impl DictExpr { impl DictExpr {
/// The named dictionary items. /// The named dictionary items.
pub fn items<'a>(&'a self) -> impl Iterator<Item = Named> + 'a { pub fn items(&self) -> impl Iterator<Item = Named> + '_ {
self.0.children().filter_map(RedRef::cast) self.0.children().filter_map(RedRef::cast)
} }
} }
@ -439,7 +451,7 @@ node! {
impl BlockExpr { impl BlockExpr {
/// The list of expressions contained in the block. /// The list of expressions contained in the block.
pub fn exprs<'a>(&'a self) -> impl Iterator<Item = Expr> + 'a { pub fn exprs(&self) -> impl Iterator<Item = Expr> + '_ {
self.0.children().filter_map(RedRef::cast) self.0.children().filter_map(RedRef::cast)
} }
} }
@ -477,7 +489,7 @@ pub enum UnOp {
} }
impl TypedNode for UnOp { impl TypedNode for UnOp {
fn cast_from(node: RedRef) -> Option<Self> { fn from_red(node: RedRef) -> Option<Self> {
Self::from_token(node.kind()) Self::from_token(node.kind())
} }
} }
@ -581,7 +593,7 @@ pub enum BinOp {
} }
impl TypedNode for BinOp { impl TypedNode for BinOp {
fn cast_from(node: RedRef) -> Option<Self> { fn from_red(node: RedRef) -> Option<Self> {
Self::from_token(node.kind()) Self::from_token(node.kind())
} }
} }
@ -709,7 +721,7 @@ node! {
impl CallArgs { impl CallArgs {
/// The positional and named arguments. /// The positional and named arguments.
pub fn items<'a>(&'a self) -> impl Iterator<Item = CallArg> + 'a { pub fn items(&self) -> impl Iterator<Item = CallArg> + '_ {
self.0.children().filter_map(RedRef::cast) self.0.children().filter_map(RedRef::cast)
} }
} }
@ -726,7 +738,7 @@ pub enum CallArg {
} }
impl TypedNode for CallArg { impl TypedNode for CallArg {
fn cast_from(node: RedRef) -> Option<Self> { fn from_red(node: RedRef) -> Option<Self> {
match node.kind() { match node.kind() {
NodeKind::Named => Some(CallArg::Named( NodeKind::Named => Some(CallArg::Named(
node.cast().expect("named call argument is missing name"), node.cast().expect("named call argument is missing name"),
@ -767,7 +779,7 @@ impl ClosureExpr {
} }
/// The parameter bindings. /// The parameter bindings.
pub fn params<'a>(&'a self) -> impl Iterator<Item = ClosureParam> + 'a { pub fn params(&self) -> impl Iterator<Item = ClosureParam> + '_ {
self.0 self.0
.children() .children()
.find(|x| x.kind() == &NodeKind::ClosureParams) .find(|x| x.kind() == &NodeKind::ClosureParams)
@ -805,10 +817,10 @@ pub enum ClosureParam {
} }
impl TypedNode for ClosureParam { impl TypedNode for ClosureParam {
fn cast_from(node: RedRef) -> Option<Self> { fn from_red(node: RedRef) -> Option<Self> {
match node.kind() { match node.kind() {
NodeKind::Ident(i) => { NodeKind::Ident(id) => {
Some(ClosureParam::Pos(Ident::new(i, node.span()).unwrap())) Some(ClosureParam::Pos(Ident::new_unchecked(id, node.span())))
} }
NodeKind::Named => Some(ClosureParam::Named( NodeKind::Named => Some(ClosureParam::Named(
node.cast().expect("named closure parameter is missing name"), node.cast().expect("named closure parameter is missing name"),
@ -921,7 +933,7 @@ pub enum Imports {
} }
impl TypedNode for Imports { impl TypedNode for Imports {
fn cast_from(node: RedRef) -> Option<Self> { fn from_red(node: RedRef) -> Option<Self> {
match node.kind() { match node.kind() {
NodeKind::Star => Some(Imports::Wildcard), NodeKind::Star => Some(Imports::Wildcard),
NodeKind::ImportItems => { NodeKind::ImportItems => {
@ -1043,14 +1055,75 @@ node! {
} }
impl ForPattern { impl ForPattern {
/// The key part of the pattern: index for arrays, name for dictionaries.
pub fn key(&self) -> Option<Ident> { pub fn key(&self) -> Option<Ident> {
let mut items: Vec<_> = self.0.children().filter_map(RedRef::cast).collect(); let mut children = self.0.children().filter_map(RedRef::cast);
if items.len() > 1 { Some(items.remove(0)) } else { None } let key = children.next();
if children.next().is_some() { key } else { None }
} }
/// The value part of the pattern.
pub fn value(&self) -> Ident { pub fn value(&self) -> Ident {
self.0 self.0
.cast_last_child() .cast_last_child()
.expect("for-in loop pattern is missing value") .expect("for-in loop pattern is missing value")
} }
} }
/// An unicode identifier with a few extra permissible characters.
///
/// In addition to what is specified in the [Unicode Standard][uax31], we allow:
/// - `_` as a starting character,
/// - `_` and `-` as continuing characters.
///
/// [uax31]: http://www.unicode.org/reports/tr31/
#[derive(Debug, Clone, PartialEq)]
pub struct Ident {
/// The source code location.
pub span: Span,
/// The identifier string.
pub string: EcoString,
}
impl Ident {
/// Create a new identifier from a string checking that it is a valid.
pub fn new(
string: impl AsRef<str> + Into<EcoString>,
span: impl Into<Span>,
) -> Option<Self> {
is_ident(string.as_ref())
.then(|| Self { span: span.into(), string: string.into() })
}
/// Create a new identifier from a string and a span.
///
/// The `string` must be a valid identifier.
#[track_caller]
pub fn new_unchecked(string: impl Into<EcoString>, span: Span) -> Self {
let string = string.into();
debug_assert!(is_ident(&string), "`{}` is not a valid identifier", string);
Self { span, string }
}
/// Return a reference to the underlying string.
pub fn as_str(&self) -> &str {
&self.string
}
}
impl Deref for Ident {
type Target = str;
fn deref(&self) -> &Self::Target {
self.as_str()
}
}
impl TypedNode for Ident {
fn from_red(node: RedRef) -> Option<Self> {
match node.kind() {
NodeKind::Ident(string) => Some(Ident::new_unchecked(string, node.span())),
_ => None,
}
}
}

94
src/syntax/ident.rs
View File

@ -1,94 +0,0 @@
use std::borrow::Borrow;
use std::ops::Deref;
use unicode_xid::UnicodeXID;
use super::{NodeKind, RedRef, Span, TypedNode};
use crate::util::EcoString;
/// An unicode identifier with a few extra permissible characters.
///
/// In addition to what is specified in the [Unicode Standard][uax31], we allow:
/// - `_` as a starting character,
/// - `_` and `-` as continuing characters.
///
/// [uax31]: http://www.unicode.org/reports/tr31/
#[derive(Debug, Clone, PartialEq)]
pub struct Ident {
/// The source code location.
pub span: Span,
/// The identifier string.
pub string: EcoString,
}
impl Ident {
/// Create a new identifier from a string checking that it is a valid.
pub fn new(
string: impl AsRef<str> + Into<EcoString>,
span: impl Into<Span>,
) -> Option<Self> {
if is_ident(string.as_ref()) {
Some(Self { span: span.into(), string: string.into() })
} else {
None
}
}
/// Return a reference to the underlying string.
pub fn as_str(&self) -> &str {
self
}
}
impl Deref for Ident {
type Target = str;
fn deref(&self) -> &Self::Target {
self.string.as_str()
}
}
impl AsRef<str> for Ident {
fn as_ref(&self) -> &str {
self
}
}
impl Borrow<str> for Ident {
fn borrow(&self) -> &str {
self
}
}
impl From<&Ident> for EcoString {
fn from(ident: &Ident) -> Self {
ident.string.clone()
}
}
impl TypedNode for Ident {
fn cast_from(node: RedRef) -> Option<Self> {
match node.kind() {
NodeKind::Ident(i) => Some(Ident::new(i, node.span()).unwrap()),
_ => None,
}
}
}
/// Whether a string is a valid identifier.
pub fn is_ident(string: &str) -> bool {
let mut chars = string.chars();
chars
.next()
.map_or(false, |c| is_id_start(c) && chars.all(is_id_continue))
}
/// Whether a character can start an identifier.
pub fn is_id_start(c: char) -> bool {
c.is_xid_start() || c == '_'
}
/// Whether a character can continue an identifier.
pub fn is_id_continue(c: char) -> bool {
c.is_xid_continue() || c == '_' || c == '-'
}

197
src/syntax/mod.rs
View File

@ -1,31 +1,28 @@
//! Syntax types. //! Syntax types.
mod ast; pub mod ast;
mod ident;
mod pretty; mod pretty;
mod span; mod span;
use std::fmt; use std::fmt::{self, Debug, Display, Formatter};
use std::fmt::{Debug, Display, Formatter};
use std::mem; use std::mem;
use std::rc::Rc; use std::rc::Rc;
pub use ast::*;
pub use ident::*;
pub use pretty::*; pub use pretty::*;
pub use span::*; pub use span::*;
use self::ast::TypedNode;
use crate::diag::Error; use crate::diag::Error;
use crate::geom::{AngularUnit, LengthUnit}; use crate::geom::{AngularUnit, LengthUnit};
use crate::source::SourceId; use crate::source::SourceId;
use crate::util::EcoString; use crate::util::EcoString;
/// Children of a [`GreenNode`]. /// An inner of leaf node in the untyped green tree.
#[derive(Clone, PartialEq)] #[derive(Clone, PartialEq)]
pub enum Green { pub enum Green {
/// A non-terminal node in an Rc. /// A reference-counted inner node.
Node(Rc<GreenNode>), Node(Rc<GreenNode>),
/// A terminal owned token. /// A terminal, owned token.
Token(GreenData), Token(GreenData),
} }
@ -77,13 +74,12 @@ impl Debug for Green {
f.debug_list().entries(&n.children).finish()?; f.debug_list().entries(&n.children).finish()?;
} }
} }
Ok(()) Ok(())
} }
} }
/// A syntactical node. /// An inner node in the untyped green tree.
#[derive(Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub struct GreenNode { pub struct GreenNode {
/// Node metadata. /// Node metadata.
data: GreenData, data: GreenData,
@ -122,15 +118,15 @@ impl From<Rc<GreenNode>> for Green {
} }
} }
/// Data shared between [`GreenNode`]s and leaf nodes. /// Data shared between inner and leaf nodes.
#[derive(Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub struct GreenData { pub struct GreenData {
/// What kind of node this is (each kind would have its own struct in a /// What kind of node this is (each kind would have its own struct in a
/// strongly typed AST). /// strongly typed AST).
kind: NodeKind, kind: NodeKind,
/// The byte length of the node in the source. /// The byte length of the node in the source.
len: usize, len: usize,
/// Whether this node or any of its children are erroneous. /// Whether this node or any of its children contain an error.
erroneous: bool, erroneous: bool,
} }
@ -162,8 +158,9 @@ impl From<GreenData> for Green {
} }
} }
/// A borrowed wrapper for the [`GreenNode`] type that allows to access spans, /// A borrowed wrapper for a [`GreenNode`] with span information.
/// error lists and cast to an AST. ///
/// Borrowed variant of [`RedNode`]. Can be [cast](Self::cast) to an AST node.
#[derive(Copy, Clone, PartialEq)] #[derive(Copy, Clone, PartialEq)]
pub struct RedRef<'a> { pub struct RedRef<'a> {
id: SourceId, id: SourceId,
@ -182,50 +179,27 @@ impl<'a> RedRef<'a> {
} }
/// The type of the node. /// The type of the node.
pub fn kind(&self) -> &NodeKind { pub fn kind(self) -> &'a NodeKind {
self.green.kind() self.green.kind()
} }
/// The span of the node.
pub fn span(&self) -> Span {
Span::new(self.id, self.offset, self.offset + self.green.len())
}
/// The length of the node. /// The length of the node.
pub fn len(&self) -> usize { pub fn len(self) -> usize {
self.green.len() self.green.len()
} }
/// Convert the node to a typed AST node. /// The span of the node.
pub fn cast<T>(self) -> Option<T> pub fn span(self) -> Span {
where Span::new(self.id, self.offset, self.offset + self.green.len())
T: TypedNode,
{
T::cast_from(self)
} }
/// Whether the node or its children contain an error. /// Whether the node or its children contain an error.
pub fn erroneous(&self) -> bool { pub fn erroneous(self) -> bool {
self.green.erroneous() self.green.erroneous()
} }
/// The node's children.
pub fn children(self) -> impl Iterator<Item = RedRef<'a>> + Clone {
let children = match &self.green {
Green::Node(node) => node.children(),
Green::Token(_) => &[],
};
let mut offset = self.offset;
children.iter().map(move |green| {
let child_offset = offset;
offset += green.len();
RedRef { id: self.id, offset: child_offset, green }
})
}
/// The error messages for this node and its descendants. /// The error messages for this node and its descendants.
pub fn errors(&self) -> Vec<Error> { pub fn errors(self) -> Vec<Error> {
if !self.green.erroneous() { if !self.green.erroneous() {
return vec![]; return vec![];
} }
@ -248,19 +222,42 @@ impl<'a> RedRef<'a> {
} }
} }
/// Convert the node to a typed AST node.
pub fn cast<T>(self) -> Option<T>
where
T: TypedNode,
{
T::from_red(self)
}
/// The node's children.
pub fn children(self) -> impl Iterator<Item = RedRef<'a>> {
let children = match &self.green {
Green::Node(node) => node.children(),
Green::Token(_) => &[],
};
let mut offset = self.offset;
children.iter().map(move |green| {
let child_offset = offset;
offset += green.len();
RedRef { id: self.id, offset: child_offset, green }
})
}
/// Get the first child of some type. /// Get the first child of some type.
pub(crate) fn typed_child(&self, kind: &NodeKind) -> Option<RedRef> { pub(crate) fn typed_child(self, kind: &NodeKind) -> Option<RedRef<'a>> {
self.children() self.children()
.find(|x| mem::discriminant(x.kind()) == mem::discriminant(kind)) .find(|x| mem::discriminant(x.kind()) == mem::discriminant(kind))
} }
/// Get the first child that can cast to some AST type. /// Get the first child that can cast to some AST type.
pub(crate) fn cast_first_child<T: TypedNode>(&self) -> Option<T> { pub(crate) fn cast_first_child<T: TypedNode>(self) -> Option<T> {
self.children().find_map(RedRef::cast) self.children().find_map(RedRef::cast)
} }
/// Get the last child that can cast to some AST type. /// Get the last child that can cast to some AST type.
pub(crate) fn cast_last_child<T: TypedNode>(&self) -> Option<T> { pub(crate) fn cast_last_child<T: TypedNode>(self) -> Option<T> {
self.children().filter_map(RedRef::cast).last() self.children().filter_map(RedRef::cast).last()
} }
} }
@ -277,8 +274,9 @@ impl Debug for RedRef<'_> {
} }
} }
/// An owned wrapper for the [`GreenNode`] type that allows to access spans, /// A owned wrapper for a [`GreenNode`] with span information.
/// error lists and cast to an AST. ///
/// Owned variant of [`RedRef`]. Can be [cast](Self::cast) to an AST nodes.
#[derive(Clone, PartialEq)] #[derive(Clone, PartialEq)]
pub struct RedNode { pub struct RedNode {
id: SourceId, id: SourceId,
@ -293,7 +291,7 @@ impl RedNode {
} }
/// Convert to a borrowed representation. /// Convert to a borrowed representation.
pub fn as_ref<'a>(&'a self) -> RedRef<'a> { pub fn as_ref(&self) -> RedRef<'_> {
RedRef { RedRef {
id: self.id, id: self.id,
offset: self.offset, offset: self.offset,
@ -301,9 +299,9 @@ impl RedNode {
} }
} }
/// The span of the node. /// The type of the node.
pub fn span(&self) -> Span { pub fn kind(&self) -> &NodeKind {
self.as_ref().span() self.as_ref().kind()
} }
/// The length of the node. /// The length of the node.
@ -311,29 +309,29 @@ impl RedNode {
self.as_ref().len() self.as_ref().len()
} }
/// The span of the node.
pub fn span(&self) -> Span {
self.as_ref().span()
}
/// The error messages for this node and its descendants.
pub fn errors(&self) -> Vec<Error> {
self.as_ref().errors()
}
/// Convert the node to a typed AST node. /// Convert the node to a typed AST node.
pub fn cast<T>(self) -> Option<T> pub fn cast<T>(self) -> Option<T>
where where
T: TypedNode, T: TypedNode,
{ {
T::cast_from(self.as_ref()) self.as_ref().cast()
}
/// The type of the node.
pub fn kind(&self) -> &NodeKind {
self.green.kind()
} }
/// The children of the node. /// The children of the node.
pub fn children<'a>(&'a self) -> impl Iterator<Item = RedRef<'a>> + Clone { pub fn children(&self) -> impl Iterator<Item = RedRef<'_>> {
self.as_ref().children() self.as_ref().children()
} }
/// The error messages for this node and its descendants.
pub fn errors<'a>(&'a self) -> Vec<Error> {
self.as_ref().errors()
}
/// Get the first child of some type. /// Get the first child of some type.
pub(crate) fn typed_child(&self, kind: &NodeKind) -> Option<RedNode> { pub(crate) fn typed_child(&self, kind: &NodeKind) -> Option<RedNode> {
self.as_ref().typed_child(kind).map(RedRef::own) self.as_ref().typed_child(kind).map(RedRef::own)
@ -356,11 +354,10 @@ impl Debug for RedNode {
} }
} }
pub trait TypedNode: Sized { /// All syntactical building blocks that can be part of a Typst document.
/// Performs the conversion. ///
fn cast_from(value: RedRef) -> Option<Self>; /// Can be emitted as a token by the tokenizer or as part of a green node by
} /// the parser.
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub enum NodeKind { pub enum NodeKind {
/// A left square bracket: `[`. /// A left square bracket: `[`.
@ -469,7 +466,7 @@ pub enum NodeKind {
EmDash, EmDash,
/// A slash and the letter "u" followed by a hexadecimal unicode entity /// A slash and the letter "u" followed by a hexadecimal unicode entity
/// enclosed in curly braces: `\u{1F5FA}`. /// enclosed in curly braces: `\u{1F5FA}`.
UnicodeEscape(UnicodeEscapeData), UnicodeEscape(char),
/// Strong text was enabled / disabled: `*`. /// Strong text was enabled / disabled: `*`.
Strong, Strong,
/// Emphasized text was enabled / disabled: `_`. /// Emphasized text was enabled / disabled: `_`.
@ -508,12 +505,12 @@ pub enum NodeKind {
/// A percentage: `50%`. /// A percentage: `50%`.
/// ///
/// _Note_: `50%` is stored as `50.0` here, as in the corresponding /// _Note_: `50%` is stored as `50.0` here, as in the corresponding
/// [literal](Lit::Percent). /// [literal](ast::Lit::Percent).
Percentage(f64), Percentage(f64),
/// A fraction unit: `3fr`. /// A fraction unit: `3fr`.
Fraction(f64), Fraction(f64),
/// A quoted string: `"..."`. /// A quoted string: `"..."`.
Str(StrData), Str(EcoString),
/// An array expression: `(1, "hi", 12cm)`. /// An array expression: `(1, "hi", 12cm)`.
Array, Array,
/// A dictionary expression: `(thickness: 3pt, pattern: dashed)`. /// A dictionary expression: `(thickness: 3pt, pattern: dashed)`.
@ -572,24 +569,7 @@ pub enum NodeKind {
Unknown(EcoString), Unknown(EcoString),
} }
#[derive(Debug, Copy, Clone, PartialEq, Eq)] /// Payload of a raw block: `` `...` ``.
pub enum ErrorPosition {
/// At the start of the node.
Start,
/// Over the full width of the node.
Full,
/// At the end of the node.
End,
}
/// A quoted string token: `"..."`.
#[derive(Debug, Clone, PartialEq)]
pub struct StrData {
/// The string inside the quotes.
pub string: EcoString,
}
/// A raw block token: `` `...` ``.
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub struct RawData { pub struct RawData {
/// The raw text in the block. /// The raw text in the block.
@ -602,7 +582,7 @@ pub struct RawData {
pub block: bool, pub block: bool,
} }
/// A math formula token: `$2pi + x$` or `$[f'(x) = x^2]$`. /// Payload of a math formula: `$2pi + x$` or `$[f'(x) = x^2]$`.
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Clone, PartialEq)]
pub struct MathData { pub struct MathData {
/// The formula between the dollars. /// The formula between the dollars.
@ -612,17 +592,15 @@ pub struct MathData {
pub display: bool, pub display: bool,
} }
/// A unicode escape sequence token: `\u{1F5FA}`. /// Where in a node an error should be annotated.
#[derive(Debug, Clone, PartialEq)] #[derive(Debug, Copy, Clone, PartialEq, Eq)]
pub struct UnicodeEscapeData { pub enum ErrorPosition {
/// The resulting unicode character. /// At the start of the node.
pub character: char, Start,
} /// Over the full width of the node.
Full,
impl Display for NodeKind { /// At the end of the node.
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { End,
f.pad(self.as_str())
}
} }
impl NodeKind { impl NodeKind {
@ -658,6 +636,7 @@ impl NodeKind {
matches!(self, NodeKind::Error(_, _) | NodeKind::Unknown(_)) matches!(self, NodeKind::Error(_, _) | NodeKind::Unknown(_))
} }
/// A human-readable name for the kind.
pub fn as_str(&self) -> &'static str { pub fn as_str(&self) -> &'static str {
match self { match self {
Self::LeftBracket => "opening bracket", Self::LeftBracket => "opening bracket",
@ -764,3 +743,9 @@ impl NodeKind {
} }
} }
} }
impl Display for NodeKind {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
f.pad(self.as_str())
}
}

2
src/syntax/pretty.rs
View File

@ -2,7 +2,7 @@
use std::fmt::{self, Arguments, Write}; use std::fmt::{self, Arguments, Write};
use super::*; use super::ast::*;
/// Pretty print an item and return the resulting string. /// Pretty print an item and return the resulting string.
pub fn pretty<T>(item: &T) -> String pub fn pretty<T>(item: &T) -> String

130
src/syntax/span.rs
View File

@ -1,6 +1,6 @@
use std::cmp::Ordering; use std::cmp::Ordering;
use std::fmt::{self, Debug, Formatter}; use std::fmt::{self, Debug, Formatter};
use std::ops::{Add, Range}; use std::ops::Range;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
@ -53,23 +53,19 @@ pub struct Span {
/// The id of the source file. /// The id of the source file.
pub source: SourceId, pub source: SourceId,
/// The inclusive start position. /// The inclusive start position.
pub start: Pos, pub start: usize,
/// The inclusive end position. /// The inclusive end position.
pub end: Pos, pub end: usize,
} }
impl Span { impl Span {
/// Create a new span from start and end positions. /// Create a new span from start and end positions.
pub fn new(source: SourceId, start: impl Into<Pos>, end: impl Into<Pos>) -> Self { pub fn new(source: SourceId, start: usize, end: usize) -> Self {
Self { Self { source, start, end }
source,
start: start.into(),
end: end.into(),
}
} }
/// Create a span including just a single position. /// Create a span including just a single position.
pub fn at(source: SourceId, pos: impl Into<Pos> + Copy) -> Self { pub fn at(source: SourceId, pos: usize) -> Self {
Self::new(source, pos, pos) Self::new(source, pos, pos)
} }
@ -77,19 +73,29 @@ impl Span {
pub fn detached() -> Self { pub fn detached() -> Self {
Self { Self {
source: SourceId::from_raw(0), source: SourceId::from_raw(0),
start: Pos::ZERO, start: 0,
end: Pos::ZERO, end: 0,
} }
} }
/// Create a span with a different start position. /// Create a span with a different start position.
pub fn with_start(self, start: impl Into<Pos>) -> Self { pub fn with_start(self, start: usize) -> Self {
Self { start: start.into(), ..self } Self { start, ..self }
} }
/// Create a span with a different end position. /// Create a span with a different end position.
pub fn with_end(self, end: impl Into<Pos>) -> Self { pub fn with_end(self, end: usize) -> Self {
Self { end: end.into(), ..self } Self { end, ..self }
}
/// A new span at the position of this span's start.
pub fn at_start(&self) -> Span {
Self::at(self.source, self.start)
}
/// A new span at the position of this span's end.
pub fn at_end(&self) -> Span {
Self::at(self.source, self.end)
} }
/// Create a new span with the earlier start and later end position. /// Create a new span with the earlier start and later end position.
@ -110,28 +116,18 @@ impl Span {
} }
/// Test whether a position is within the span. /// Test whether a position is within the span.
pub fn contains_pos(&self, pos: Pos) -> bool { pub fn contains(&self, pos: usize) -> bool {
self.start <= pos && self.end >= pos self.start <= pos && self.end >= pos
} }
/// Test whether one span complete contains the other span. /// Test whether one span complete contains the other span.
pub fn contains(self, other: Self) -> bool { pub fn surrounds(self, other: Self) -> bool {
self.source == other.source && self.start <= other.start && self.end >= other.end self.source == other.source && self.start <= other.start && self.end >= other.end
} }
/// Convert to a `Range<Pos>` for indexing. /// Convert to a `Range<usize>` for indexing.
pub fn to_range(self) -> Range<usize> { pub fn to_range(self) -> Range<usize> {
self.start.to_usize() .. self.end.to_usize() self.start .. self.end
}
/// A new span at the position of this span's start.
pub fn at_start(&self) -> Span {
Self::at(self.source, self.start)
}
/// A new span at the position of this span's end.
pub fn at_end(&self) -> Span {
Self::at(self.source, self.end)
} }
} }
@ -150,77 +146,3 @@ impl PartialOrd for Span {
} }
} }
} }
/// A byte position in source code.
#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Serialize, Deserialize)]
pub struct Pos(pub u32);
impl Pos {
/// The zero position.
pub const ZERO: Self = Self(0);
/// Convert to a usize for indexing.
pub fn to_usize(self) -> usize {
self.0 as usize
}
}
impl Debug for Pos {
fn fmt(&self, f: &mut Formatter) -> fmt::Result {
Debug::fmt(&self.0, f)
}
}
impl From<u32> for Pos {
fn from(index: u32) -> Self {
Self(index)
}
}
impl From<usize> for Pos {
fn from(index: usize) -> Self {
Self(index as u32)
}
}
impl<T> Add<T> for Pos
where
T: Into<Pos>,
{
type Output = Self;
fn add(self, rhs: T) -> Self {
Pos(self.0 + rhs.into().0)
}
}
/// Convert a position or range into a span.
pub trait IntoSpan {
/// Convert into a span by providing the source id.
fn into_span(self, source: SourceId) -> Span;
}
impl IntoSpan for Span {
fn into_span(self, source: SourceId) -> Span {
debug_assert_eq!(self.source, source);
self
}
}
impl IntoSpan for Pos {
fn into_span(self, source: SourceId) -> Span {
Span::new(source, self, self)
}
}
impl IntoSpan for usize {
fn into_span(self, source: SourceId) -> Span {
Span::new(source, self, self)
}
}
impl IntoSpan for Range<usize> {
fn into_span(self, source: SourceId) -> Span {
Span::new(source, self.start, self.end)
}
}

14
tests/typeset.rs
View File

@ -24,7 +24,7 @@ use typst::loading::FsLoader;
use typst::parse::Scanner; use typst::parse::Scanner;
use typst::source::SourceFile; use typst::source::SourceFile;
use typst::style::Style; use typst::style::Style;
use typst::syntax::{Pos, Span}; use typst::syntax::Span;
use typst::Context; use typst::Context;
const TYP_DIR: &str = "./typ"; const TYP_DIR: &str = "./typ";
@ -355,12 +355,12 @@ fn parse_metadata(source: &SourceFile) -> (Option<bool>, Vec<Error>) {
let comments = let comments =
lines[i ..].iter().take_while(|line| line.starts_with("//")).count(); lines[i ..].iter().take_while(|line| line.starts_with("//")).count();
let pos = |s: &mut Scanner| -> Pos { let pos = |s: &mut Scanner| -> usize {
let first = num(s) - 1; let first = num(s) - 1;
let (delta, column) = let (delta, column) =
if s.eat_if(':') { (first, num(s) - 1) } else { (0, first) }; if s.eat_if(':') { (first, num(s) - 1) } else { (0, first) };
let line = (i + comments) + delta; let line = (i + comments) + delta;
source.line_column_to_byte(line, column).unwrap().into() source.line_column_to_byte(line, column).unwrap()
}; };
let mut s = Scanner::new(rest); let mut s = Scanner::new(rest);
@ -375,10 +375,10 @@ fn parse_metadata(source: &SourceFile) -> (Option<bool>, Vec<Error>) {
} }
fn print_error(source: &SourceFile, line: usize, error: &Error) { fn print_error(source: &SourceFile, line: usize, error: &Error) {
let start_line = 1 + line + source.byte_to_line(error.span.start.to_usize()).unwrap(); let start_line = 1 + line + source.byte_to_line(error.span.start).unwrap();
let start_col = 1 + source.byte_to_column(error.span.start.to_usize()).unwrap(); let start_col = 1 + source.byte_to_column(error.span.start).unwrap();
let end_line = 1 + line + source.byte_to_line(error.span.end.to_usize()).unwrap(); let end_line = 1 + line + source.byte_to_line(error.span.end).unwrap();
let end_col = 1 + source.byte_to_column(error.span.end.to_usize()).unwrap(); let end_col = 1 + source.byte_to_column(error.span.end).unwrap();
println!( println!(
"Error: {}:{}-{}:{}: {}", "Error: {}:{}-{}:{}: {}",
start_line, start_col, end_line, end_col, error.message start_line, start_col, end_line, end_col, error.message