Make syntax not depend on parse 📩

This would make it possible to split them into two separate crates.

main/src/main.rs

@@ -9,7 +9,7 @@ use futures_executor::block_on;
 
 use typstc::export::pdf;
 use typstc::font::FontLoader;
-use typstc::syntax::LineMap;
+use typstc::parse::LineMap;
 use typstc::{Feedback, Pass, Typesetter};
 
 fn main() {

src/compute/value.rs

@@ -310,7 +310,7 @@ macro_rules! impl_ident {
         impl TryFromValue for $type {
             fn try_from_value(value: Spanned<&Value>, f: &mut Feedback) -> Option<Self> {
                 if let Value::Ident(ident) = value.v {
-                    let val = $parse(ident.as_str());
+                    let val = $parse(ident);
                     if val.is_none() {
                         error!(@f, value.span, "invalid {}", $name);
                     }
@@ -352,6 +352,12 @@ impl_match!(ScaleLength, "number or length",
 /// `Into<String>`.
 pub struct StringLike(pub String);
 
+impl From<StringLike> for String {
+    fn from(like: StringLike) -> String {
+        like.0
+    }
+}
+
 impl Deref for StringLike {
     type Target = str;
 
@@ -360,12 +366,6 @@ impl Deref for StringLike {
     }
 }
 
-impl From<StringLike> for String {
-    fn from(like: StringLike) -> String {
-        like.0
-    }
-}
-
 impl_match!(StringLike, "identifier or string",
     Value::Ident(Ident(s)) => StringLike(s.clone()),
     Value::Str(s) => StringLike(s.clone()),
@@ -410,7 +410,7 @@ impl TryFromValue for FontWeight {
                 }
             }
             Value::Ident(ident) => {
-                let weight = Self::from_str(ident.as_str());
+                let weight = Self::from_str(ident);
                 if weight.is_none() {
                     error!(@f, value.span, "invalid font weight");
                 }

src/parse/lines.rs

@@ -1,9 +1,7 @@
 //! Conversion of byte positions to line/column locations.
 
-use std::fmt::{self, Debug, Display, Formatter};
+use super::Scanner;
-
+use crate::syntax::{Location, Pos};
-use super::Pos;
-use crate::parse::{is_newline_char, Scanner};
 
 /// Enables conversion of byte position to locations.
 pub struct LineMap<'s> {
@@ -18,7 +16,7 @@ impl<'s> LineMap<'s> {
         let mut s = Scanner::new(src);
 
         while let Some(c) = s.eat_merging_crlf() {
-            if is_newline_char(c) {
+            if is_newline(c) {
                 line_starts.push(s.index().into());
             }
         }
@@ -47,32 +45,14 @@ impl<'s> LineMap<'s> {
     }
 }
 
-/// One-indexed line-column position in source code.
+/// Whether this character denotes a newline.
-#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
+pub fn is_newline(character: char) -> bool {
-#[cfg_attr(feature = "serialize", derive(serde::Serialize))]
+    match character {
-pub struct Location {
+        // Line Feed, Vertical Tab, Form Feed, Carriage Return.
-    /// The one-indexed line.
+        '\n' | '\x0B' | '\x0C' | '\r' |
-    pub line: u32,
+        // Next Line, Line Separator, Paragraph Separator.
-    /// The one-indexed column.
+        '\u{0085}' | '\u{2028}' | '\u{2029}' => true,
-    pub column: u32,
+        _ => false,
-}
-
-impl Location {
-    /// Create a new location from line and column.
-    pub fn new(line: u32, column: u32) -> Self {
-        Self { line, column }
-    }
-}
-
-impl Debug for Location {
-    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
-        Display::fmt(self, f)
-    }
-}
-
-impl Display for Location {
-    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
-        write!(f, "{}:{}", self.line, self.column)
     }
 }
 

src/parse/mod.rs

@@ -1,9 +1,11 @@
 //! Parsing and tokenization.
 
+mod lines;
 mod resolve;
 mod scanner;
 mod tokens;
 
+pub use lines::*;
 pub use resolve::*;
 pub use scanner::*;
 pub use tokens::*;

src/parse/resolve.rs

@@ -1,6 +1,6 @@
 //! Resolve strings and raw blocks.
 
-use super::{is_newline_char, Scanner};
+use super::{is_newline, Scanner};
 use crate::syntax::{Ident, Raw};
 
 /// Resolves all escape sequences in a string.
@@ -42,8 +42,8 @@ pub fn resolve_string(string: &str) -> String {
     out
 }
 
-/// Resolve a hexademical escape sequence (only the inner hex letters without
+/// Resolve a hexademical escape sequence into a character
-/// braces or `\u`) into a character.
+/// (only the inner hex letters without braces or `\u`).
 pub fn resolve_hex(sequence: &str) -> Option<char> {
     u32::from_str_radix(sequence, 16).ok().and_then(std::char::from_u32)
 }
@@ -71,7 +71,7 @@ pub fn resolve_raw(raw: &str, backticks: usize) -> Raw {
 fn split_at_lang_tag(raw: &str) -> (&str, &str) {
     let mut s = Scanner::new(raw);
     (
-        s.eat_until(|c| c == '`' || c.is_whitespace() || is_newline_char(c)),
+        s.eat_until(|c| c == '`' || c.is_whitespace() || is_newline(c)),
         s.rest(),
     )
 }
@@ -101,15 +101,15 @@ fn trim_and_split_raw(raw: &str) -> (Vec<String>, bool) {
     (lines, had_newline)
 }
 
-/// Splits a string into a vector of lines (respecting Unicode & Windows line
+/// Splits a string into a vector of lines
-/// breaks).
+/// (respecting Unicode, Unix, Mac and Windows line breaks).
 pub fn split_lines(text: &str) -> Vec<String> {
     let mut s = Scanner::new(text);
     let mut line = String::new();
     let mut lines = Vec::new();
 
     while let Some(c) = s.eat_merging_crlf() {
-        if is_newline_char(c) {
+        if is_newline(c) {
             lines.push(std::mem::take(&mut line));
         } else {
             line.push(c);

src/parse/scanner.rs

@@ -102,37 +102,14 @@ impl<'s> Scanner<'s> {
     pub fn check(&self, f: impl FnMut(char) -> bool) -> bool {
         self.peek().map(f).unwrap_or(false)
     }
+
+    /// Go back to the where the index says.
+    fn reset(&mut self) {
+        self.iter = self.src[self.index ..].chars();
+    }
 }
 
 impl<'s> Scanner<'s> {
-    /// Slice a part out of the source string.
-    pub fn get<I>(&self, index: I) -> &'s str
-    where
-        I: SliceIndex<str, Output = str>,
-    {
-        &self.src[index]
-    }
-
-    /// The full source string.
-    pub fn src(&self) -> &'s str {
-        self.src
-    }
-
-    /// The full string up to the current index.
-    pub fn eaten(&self) -> &'s str {
-        &self.src[.. self.index]
-    }
-
-    /// The string from `start` to the current index.
-    pub fn eaten_from(&self, start: usize) -> &'s str {
-        &self.src[start .. self.index]
-    }
-
-    /// The remaining string after the current index.
-    pub fn rest(&self) -> &'s str {
-        &self.src[self.index ..]
-    }
-
     /// The current index in the string.
     pub fn index(&self) -> usize {
         self.index
@@ -147,9 +124,32 @@ impl<'s> Scanner<'s> {
             .unwrap_or(0)
     }
 
-    /// Go back to the where the index says.
+    /// Slice a part out of the source string.
-    fn reset(&mut self) {
+    pub fn get<I>(&self, index: I) -> &'s str
-        self.iter = self.src[self.index ..].chars();
+    where
+        I: SliceIndex<str, Output = str>,
+    {
+        &self.src[index]
+    }
+
+    /// The full source string.
+    pub fn src(&self) -> &'s str {
+        self.src
+    }
+
+    /// The full source string up to the current index.
+    pub fn eaten(&self) -> &'s str {
+        &self.src[.. self.index]
+    }
+
+    /// The source string from `start` to the current index.
+    pub fn eaten_from(&self, start: usize) -> &'s str {
+        &self.src[start .. self.index]
+    }
+
+    /// The remaining source string after the current index.
+    pub fn rest(&self) -> &'s str {
+        &self.src[self.index ..]
     }
 }
 
@@ -158,14 +158,3 @@ impl Debug for Scanner<'_> {
         write!(f, "Scanner({}|{})", self.eaten(), self.rest())
     }
 }
-
-/// Whether this character denotes a newline.
-pub fn is_newline_char(character: char) -> bool {
-    match character {
-        // Line Feed, Vertical Tab, Form Feed, Carriage Return.
-        '\n' | '\x0B' | '\x0C' | '\r' |
-        // Next Line, Line Separator, Paragraph Separator.
-        '\u{0085}' | '\u{2028}' | '\u{2029}' => true,
-        _ => false,
-    }
-}

src/parse/tokens.rs

@@ -1,8 +1,8 @@
 //! Tokenization.
 
-use super::{is_newline_char, Scanner};
+use super::{is_newline, Scanner};
 use crate::length::Length;
-use crate::syntax::{Ident, Pos, Span, SpanWith, Spanned, Token};
+use crate::syntax::{is_ident, Pos, Span, SpanWith, Spanned, Token};
 
 use TokenMode::*;
 
@@ -115,7 +115,7 @@ impl<'s> Tokens<'s> {
 
         // Uneat the first char if it's a newline, so that it's counted in the
         // loop.
-        if is_newline_char(first) {
+        if is_newline(first) {
             self.s.uneat();
         }
 
@@ -127,7 +127,7 @@ impl<'s> Tokens<'s> {
                 break;
             }
 
-            if is_newline_char(c) {
+            if is_newline(c) {
                 newlines += 1;
             }
         }
@@ -136,7 +136,7 @@ impl<'s> Tokens<'s> {
     }
 
     fn read_line_comment(&mut self) -> Token<'s> {
-        Token::LineComment(self.s.eat_until(is_newline_char))
+        Token::LineComment(self.s.eat_until(is_newline))
     }
 
     fn read_block_comment(&mut self) -> Token<'s> {
@@ -277,7 +277,7 @@ fn parse_expr(text: &str) -> Token<'_> {
         Token::Number(num / 100.0)
     } else if let Ok(length) = text.parse::<Length>() {
         Token::Length(length)
-    } else if Ident::is_ident(text) {
+    } else if is_ident(text) {
         Token::Ident(text)
     } else {
         Token::Invalid(text)

src/syntax/mod.rs

@@ -1,11 +1,9 @@
 //! Syntax types.
 
-mod lines;
 mod span;
 mod token;
 mod tree;
 
-pub use lines::*;
 pub use span::*;
 pub use token::*;
 pub use tree::*;

src/syntax/span.rs

@@ -1,6 +1,6 @@
 //! Mapping of values to the locations they originate from in source code.
 
-use std::fmt::{self, Debug, Formatter};
+use std::fmt::{self, Debug, Display, Formatter};
 
 #[cfg(test)]
 use std::cell::Cell;
@@ -168,7 +168,7 @@ impl Debug for Span {
     }
 }
 
-/// A byte position.
+/// A byte position in source code.
 #[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
 #[cfg_attr(feature = "serialize", derive(serde::Serialize))]
 pub struct Pos(pub u32);
@@ -203,6 +203,35 @@ impl Offset for Pos {
 
 impl Debug for Pos {
     fn fmt(&self, f: &mut Formatter) -> fmt::Result {
-        self.0.fmt(f)
+        Debug::fmt(&self.0, f)
+    }
+}
+
+/// A one-indexed line-column position in source code.
+#[derive(Copy, Clone, Eq, PartialEq, Ord, PartialOrd, Hash)]
+#[cfg_attr(feature = "serialize", derive(serde::Serialize))]
+pub struct Location {
+    /// The one-indexed line.
+    pub line: u32,
+    /// The one-indexed column.
+    pub column: u32,
+}
+
+impl Location {
+    /// Create a new location from line and column.
+    pub fn new(line: u32, column: u32) -> Self {
+        Self { line, column }
+    }
+}
+
+impl Debug for Location {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        Display::fmt(self, f)
+    }
+}
+
+impl Display for Location {
+    fn fmt(&self, f: &mut Formatter) -> fmt::Result {
+        write!(f, "{}:{}", self.line, self.column)
     }
 }

src/syntax/tree.rs

@@ -1,6 +1,7 @@
 //! The syntax tree.
 
 use std::fmt::{self, Debug, Formatter};
+use std::ops::Deref;
 
 use unicode_xid::UnicodeXID;
 
@@ -234,7 +235,7 @@ pub struct Ident(pub String);
 impl Ident {
     /// Create a new identifier from a string checking that it is a valid.
     pub fn new(ident: impl AsRef<str> + Into<String>) -> Option<Self> {
-        if Self::is_ident(ident.as_ref()) {
+        if is_ident(ident.as_ref()) {
             Some(Self(ident.into()))
         } else {
             None
@@ -243,21 +244,21 @@ impl Ident {
 
     /// Return a reference to the underlying string.
     pub fn as_str(&self) -> &str {
-        self.0.as_str()
+        self
     }
+}
 
-    /// Whether the string is a valid identifier.
+impl AsRef<str> for Ident {
-    pub fn is_ident(string: &str) -> bool {
+    fn as_ref(&self) -> &str {
-        fn is_ok(c: char) -> bool {
+        self
-            c == '-' || c == '_'
+    }
-        }
+}
 
-        let mut chars = string.chars();
+impl Deref for Ident {
-        if matches!(chars.next(), Some(c) if c.is_xid_start() || is_ok(c)) {
+    type Target = str;
-            chars.all(|c| c.is_xid_continue() || is_ok(c))
+
-        } else {
+    fn deref(&self) -> &Self::Target {
-            false
+        self.0.as_str()
-        }
     }
 }
 
@@ -267,6 +268,20 @@ impl Debug for Ident {
     }
 }
 
+/// Whether the string is a valid identifier.
+pub fn is_ident(string: &str) -> bool {
+    fn is_ok(c: char) -> bool {
+        c == '-' || c == '_'
+    }
+
+    let mut chars = string.chars();
+    if matches!(chars.next(), Some(c) if c.is_xid_start() || is_ok(c)) {
+        chars.all(|c| c.is_xid_continue() || is_ok(c))
+    } else {
+        false
+    }
+}
+
 /// A table of expressions.
 ///
 /// # Example
@@ -307,7 +322,7 @@ pub struct CallExpr {
 impl CallExpr {
     /// Evaluate the call expression to a value.
     pub async fn eval(&self, ctx: &LayoutContext<'_>, f: &mut Feedback) -> Value {
-        let name = self.name.v.as_str();
+        let name = &self.name.v;
         let span = self.name.span;
         let args = self.args.eval(ctx, f).await;
 

tests/test_typeset.rs

@@ -18,8 +18,8 @@ use typstc::layout::elements::{LayoutElement, Shaped};
 use typstc::layout::MultiLayout;
 use typstc::length::Length;
 use typstc::paper::PaperClass;
+use typstc::parse::LineMap;
 use typstc::style::PageStyle;
-use typstc::syntax::LineMap;
 use typstc::{Feedback, Pass, Typesetter};
 
 const TEST_DIR: &str = "tests";