diff --git a/main/src/main.rs b/main/src/main.rs index 5b3538d59..2a09c77c4 100644 --- a/main/src/main.rs +++ b/main/src/main.rs @@ -9,7 +9,7 @@ use futures_executor::block_on; use typstc::export::pdf; use typstc::font::FontLoader; -use typstc::syntax::LineMap; +use typstc::parse::LineMap; use typstc::{Feedback, Pass, Typesetter}; fn main() { diff --git a/src/compute/value.rs b/src/compute/value.rs index ef36e5d8a..e8e4cbe1f 100644 --- a/src/compute/value.rs +++ b/src/compute/value.rs @@ -310,7 +310,7 @@ macro_rules! impl_ident { impl TryFromValue for $type { fn try_from_value(value: Spanned<&Value>, f: &mut Feedback) -> Option { if let Value::Ident(ident) = value.v { - let val = $parse(ident.as_str()); + let val = $parse(ident); if val.is_none() { error!(@f, value.span, "invalid {}", $name); } @@ -352,6 +352,12 @@ impl_match!(ScaleLength, "number or length", /// `Into`. pub struct StringLike(pub String); +impl From for String { + fn from(like: StringLike) -> String { + like.0 + } +} + impl Deref for StringLike { type Target = str; @@ -360,12 +366,6 @@ impl Deref for StringLike { } } -impl From for String { - fn from(like: StringLike) -> String { - like.0 - } -} - impl_match!(StringLike, "identifier or string", Value::Ident(Ident(s)) => StringLike(s.clone()), Value::Str(s) => StringLike(s.clone()), @@ -410,7 +410,7 @@ impl TryFromValue for FontWeight { } } Value::Ident(ident) => { - let weight = Self::from_str(ident.as_str()); + let weight = Self::from_str(ident); if weight.is_none() { error!(@f, value.span, "invalid font weight"); } diff --git a/src/syntax/lines.rs b/src/parse/lines.rs similarity index 71% rename from src/syntax/lines.rs rename to src/parse/lines.rs index 6ea223c44..ce5a1fe5a 100644 --- a/src/syntax/lines.rs +++ b/src/parse/lines.rs @@ -1,9 +1,7 @@ //! Conversion of byte positions to line/column locations. -use std::fmt::{self, Debug, Display, Formatter}; - -use super::Pos; -use crate::parse::{is_newline_char, Scanner}; +use super::Scanner; +use crate::syntax::{Location, Pos}; /// Enables conversion of byte position to locations. pub struct LineMap<'s> { @@ -18,7 +16,7 @@ impl<'s> LineMap<'s> { let mut s = Scanner::new(src); while let Some(c) = s.eat_merging_crlf() { - if is_newline_char(c) { + if is_newline(c) { line_starts.push(s.index().into()); } } @@ -47,32 +45,14 @@ impl<'s> LineMap<'s> { } } -/// One-indexed line-column position in source code. -#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] -#[cfg_attr(feature = "serialize", derive(serde::Serialize))] -pub struct Location { - /// The one-indexed line. - pub line: u32, - /// The one-indexed column. - pub column: u32, -} - -impl Location { - /// Create a new location from line and column. - pub fn new(line: u32, column: u32) -> Self { - Self { line, column } - } -} - -impl Debug for Location { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - Display::fmt(self, f) - } -} - -impl Display for Location { - fn fmt(&self, f: &mut Formatter) -> fmt::Result { - write!(f, "{}:{}", self.line, self.column) +/// Whether this character denotes a newline. +pub fn is_newline(character: char) -> bool { + match character { + // Line Feed, Vertical Tab, Form Feed, Carriage Return. + '\n' | '\x0B' | '\x0C' | '\r' | + // Next Line, Line Separator, Paragraph Separator. + '\u{0085}' | '\u{2028}' | '\u{2029}' => true, + _ => false, } } diff --git a/src/parse/mod.rs b/src/parse/mod.rs index 8c879d124..b62bd5d31 100644 --- a/src/parse/mod.rs +++ b/src/parse/mod.rs @@ -1,9 +1,11 @@ //! Parsing and tokenization. +mod lines; mod resolve; mod scanner; mod tokens; +pub use lines::*; pub use resolve::*; pub use scanner::*; pub use tokens::*; diff --git a/src/parse/resolve.rs b/src/parse/resolve.rs index 0e2ebd4bc..d4babd259 100644 --- a/src/parse/resolve.rs +++ b/src/parse/resolve.rs @@ -1,6 +1,6 @@ //! Resolve strings and raw blocks. -use super::{is_newline_char, Scanner}; +use super::{is_newline, Scanner}; use crate::syntax::{Ident, Raw}; /// Resolves all escape sequences in a string. @@ -42,8 +42,8 @@ pub fn resolve_string(string: &str) -> String { out } -/// Resolve a hexademical escape sequence (only the inner hex letters without -/// braces or `\u`) into a character. +/// Resolve a hexademical escape sequence into a character +/// (only the inner hex letters without braces or `\u`). pub fn resolve_hex(sequence: &str) -> Option { u32::from_str_radix(sequence, 16).ok().and_then(std::char::from_u32) } @@ -71,7 +71,7 @@ pub fn resolve_raw(raw: &str, backticks: usize) -> Raw { fn split_at_lang_tag(raw: &str) -> (&str, &str) { let mut s = Scanner::new(raw); ( - s.eat_until(|c| c == '`' || c.is_whitespace() || is_newline_char(c)), + s.eat_until(|c| c == '`' || c.is_whitespace() || is_newline(c)), s.rest(), ) } @@ -101,15 +101,15 @@ fn trim_and_split_raw(raw: &str) -> (Vec, bool) { (lines, had_newline) } -/// Splits a string into a vector of lines (respecting Unicode & Windows line -/// breaks). +/// Splits a string into a vector of lines +/// (respecting Unicode, Unix, Mac and Windows line breaks). pub fn split_lines(text: &str) -> Vec { let mut s = Scanner::new(text); let mut line = String::new(); let mut lines = Vec::new(); while let Some(c) = s.eat_merging_crlf() { - if is_newline_char(c) { + if is_newline(c) { lines.push(std::mem::take(&mut line)); } else { line.push(c); diff --git a/src/parse/scanner.rs b/src/parse/scanner.rs index 1bffc2041..9447222d9 100644 --- a/src/parse/scanner.rs +++ b/src/parse/scanner.rs @@ -102,37 +102,14 @@ impl<'s> Scanner<'s> { pub fn check(&self, f: impl FnMut(char) -> bool) -> bool { self.peek().map(f).unwrap_or(false) } + + /// Go back to the where the index says. + fn reset(&mut self) { + self.iter = self.src[self.index ..].chars(); + } } impl<'s> Scanner<'s> { - /// Slice a part out of the source string. - pub fn get(&self, index: I) -> &'s str - where - I: SliceIndex, - { - &self.src[index] - } - - /// The full source string. - pub fn src(&self) -> &'s str { - self.src - } - - /// The full string up to the current index. - pub fn eaten(&self) -> &'s str { - &self.src[.. self.index] - } - - /// The string from `start` to the current index. - pub fn eaten_from(&self, start: usize) -> &'s str { - &self.src[start .. self.index] - } - - /// The remaining string after the current index. - pub fn rest(&self) -> &'s str { - &self.src[self.index ..] - } - /// The current index in the string. pub fn index(&self) -> usize { self.index @@ -147,9 +124,32 @@ impl<'s> Scanner<'s> { .unwrap_or(0) } - /// Go back to the where the index says. - fn reset(&mut self) { - self.iter = self.src[self.index ..].chars(); + /// Slice a part out of the source string. + pub fn get(&self, index: I) -> &'s str + where + I: SliceIndex, + { + &self.src[index] + } + + /// The full source string. + pub fn src(&self) -> &'s str { + self.src + } + + /// The full source string up to the current index. + pub fn eaten(&self) -> &'s str { + &self.src[.. self.index] + } + + /// The source string from `start` to the current index. + pub fn eaten_from(&self, start: usize) -> &'s str { + &self.src[start .. self.index] + } + + /// The remaining source string after the current index. + pub fn rest(&self) -> &'s str { + &self.src[self.index ..] } } @@ -158,14 +158,3 @@ impl Debug for Scanner<'_> { write!(f, "Scanner({}|{})", self.eaten(), self.rest()) } } - -/// Whether this character denotes a newline. -pub fn is_newline_char(character: char) -> bool { - match character { - // Line Feed, Vertical Tab, Form Feed, Carriage Return. - '\n' | '\x0B' | '\x0C' | '\r' | - // Next Line, Line Separator, Paragraph Separator. - '\u{0085}' | '\u{2028}' | '\u{2029}' => true, - _ => false, - } -} diff --git a/src/parse/tokens.rs b/src/parse/tokens.rs index cdb92c597..9f30f5876 100644 --- a/src/parse/tokens.rs +++ b/src/parse/tokens.rs @@ -1,8 +1,8 @@ //! Tokenization. -use super::{is_newline_char, Scanner}; +use super::{is_newline, Scanner}; use crate::length::Length; -use crate::syntax::{Ident, Pos, Span, SpanWith, Spanned, Token}; +use crate::syntax::{is_ident, Pos, Span, SpanWith, Spanned, Token}; use TokenMode::*; @@ -115,7 +115,7 @@ impl<'s> Tokens<'s> { // Uneat the first char if it's a newline, so that it's counted in the // loop. - if is_newline_char(first) { + if is_newline(first) { self.s.uneat(); } @@ -127,7 +127,7 @@ impl<'s> Tokens<'s> { break; } - if is_newline_char(c) { + if is_newline(c) { newlines += 1; } } @@ -136,7 +136,7 @@ impl<'s> Tokens<'s> { } fn read_line_comment(&mut self) -> Token<'s> { - Token::LineComment(self.s.eat_until(is_newline_char)) + Token::LineComment(self.s.eat_until(is_newline)) } fn read_block_comment(&mut self) -> Token<'s> { @@ -277,7 +277,7 @@ fn parse_expr(text: &str) -> Token<'_> { Token::Number(num / 100.0) } else if let Ok(length) = text.parse::() { Token::Length(length) - } else if Ident::is_ident(text) { + } else if is_ident(text) { Token::Ident(text) } else { Token::Invalid(text) diff --git a/src/syntax/mod.rs b/src/syntax/mod.rs index f442ba9e0..fe887c2fe 100644 --- a/src/syntax/mod.rs +++ b/src/syntax/mod.rs @@ -1,11 +1,9 @@ //! Syntax types. -mod lines; mod span; mod token; mod tree; -pub use lines::*; pub use span::*; pub use token::*; pub use tree::*; diff --git a/src/syntax/span.rs b/src/syntax/span.rs index d803eeebf..eb029479f 100644 --- a/src/syntax/span.rs +++ b/src/syntax/span.rs @@ -1,6 +1,6 @@ //! Mapping of values to the locations they originate from in source code. -use std::fmt::{self, Debug, Formatter}; +use std::fmt::{self, Debug, Display, Formatter}; #[cfg(test)] use std::cell::Cell; @@ -168,7 +168,7 @@ impl Debug for Span { } } -/// A byte position. +/// A byte position in source code. #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] #[cfg_attr(feature = "serialize", derive(serde::Serialize))] pub struct Pos(pub u32); @@ -203,6 +203,35 @@ impl Offset for Pos { impl Debug for Pos { fn fmt(&self, f: &mut Formatter) -> fmt::Result { - self.0.fmt(f) + Debug::fmt(&self.0, f) + } +} + +/// A one-indexed line-column position in source code. +#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)] +#[cfg_attr(feature = "serialize", derive(serde::Serialize))] +pub struct Location { + /// The one-indexed line. + pub line: u32, + /// The one-indexed column. + pub column: u32, +} + +impl Location { + /// Create a new location from line and column. + pub fn new(line: u32, column: u32) -> Self { + Self { line, column } + } +} + +impl Debug for Location { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + Display::fmt(self, f) + } +} + +impl Display for Location { + fn fmt(&self, f: &mut Formatter) -> fmt::Result { + write!(f, "{}:{}", self.line, self.column) } } diff --git a/src/syntax/tree.rs b/src/syntax/tree.rs index bfbb3706d..ee53d476c 100644 --- a/src/syntax/tree.rs +++ b/src/syntax/tree.rs @@ -1,6 +1,7 @@ //! The syntax tree. use std::fmt::{self, Debug, Formatter}; +use std::ops::Deref; use unicode_xid::UnicodeXID; @@ -234,7 +235,7 @@ pub struct Ident(pub String); impl Ident { /// Create a new identifier from a string checking that it is a valid. pub fn new(ident: impl AsRef + Into) -> Option { - if Self::is_ident(ident.as_ref()) { + if is_ident(ident.as_ref()) { Some(Self(ident.into())) } else { None @@ -243,21 +244,21 @@ impl Ident { /// Return a reference to the underlying string. pub fn as_str(&self) -> &str { - self.0.as_str() + self } +} - /// Whether the string is a valid identifier. - pub fn is_ident(string: &str) -> bool { - fn is_ok(c: char) -> bool { - c == '-' || c == '_' - } +impl AsRef for Ident { + fn as_ref(&self) -> &str { + self + } +} - let mut chars = string.chars(); - if matches!(chars.next(), Some(c) if c.is_xid_start() || is_ok(c)) { - chars.all(|c| c.is_xid_continue() || is_ok(c)) - } else { - false - } +impl Deref for Ident { + type Target = str; + + fn deref(&self) -> &Self::Target { + self.0.as_str() } } @@ -267,6 +268,20 @@ impl Debug for Ident { } } +/// Whether the string is a valid identifier. +pub fn is_ident(string: &str) -> bool { + fn is_ok(c: char) -> bool { + c == '-' || c == '_' + } + + let mut chars = string.chars(); + if matches!(chars.next(), Some(c) if c.is_xid_start() || is_ok(c)) { + chars.all(|c| c.is_xid_continue() || is_ok(c)) + } else { + false + } +} + /// A table of expressions. /// /// # Example @@ -307,7 +322,7 @@ pub struct CallExpr { impl CallExpr { /// Evaluate the call expression to a value. pub async fn eval(&self, ctx: &LayoutContext<'_>, f: &mut Feedback) -> Value { - let name = self.name.v.as_str(); + let name = &self.name.v; let span = self.name.span; let args = self.args.eval(ctx, f).await; diff --git a/tests/test_typeset.rs b/tests/test_typeset.rs index 2c8a093d3..b81cecf6c 100644 --- a/tests/test_typeset.rs +++ b/tests/test_typeset.rs @@ -18,8 +18,8 @@ use typstc::layout::elements::{LayoutElement, Shaped}; use typstc::layout::MultiLayout; use typstc::length::Length; use typstc::paper::PaperClass; +use typstc::parse::LineMap; use typstc::style::PageStyle; -use typstc::syntax::LineMap; use typstc::{Feedback, Pass, Typesetter}; const TEST_DIR: &str = "tests";