Asyncify font loading 🪐

Cargo.toml

@@ -6,20 +6,28 @@ edition = "2018"
 build = "build.rs"
 
 [dependencies]
+toddle = { path = "../toddle", default-features = false }
 tide = { path = "../tide" }
-toddle = { path = "../toddle" }
 byteorder = "1"
 smallvec = "0.6.10"
 unicode-xid = "0.1.0"
+async-trait = "0.1.22"
+futures-executor = { version = "0.3", optional = true }
+
+[features]
+default = ["fs-provider", "futures-executor"]
+fs-provider = ["toddle/fs-provider"]
 
 [[bin]]
 name = "typst-bin"
 path = "src/bin/main.rs"
+required-features = ["futures-executor"]
 
 [[test]]
 name = "layout"
 path = "tests/layout.rs"
 harness = false
+required-features = ["futures-executor"]
 
 [[test]]
 name = "parse"

src/bin/main.rs

@@ -2,6 +2,8 @@ use std::fs::{File, read_to_string};
 use std::io::BufWriter;
 use std::path::{Path, PathBuf};
 
+use futures_executor::block_on;
+
 use typstc::Typesetter;
 use typstc::toddle::query::FileSystemFontProvider;
 use typstc::export::pdf::PdfExporter;
@@ -39,7 +41,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
     let provider = FileSystemFontProvider::from_index("../fonts/index.json").unwrap();
     typesetter.add_font_provider(provider);
 
-    let layouts = typesetter.typeset(&src)?;
+    let layouts = block_on(typesetter.typeset(&src))?;
 
     let exporter = PdfExporter::new();
     let writer = BufWriter::new(File::create(&dest)?);

src/func/macros.rs

@@ -108,9 +108,20 @@ macro_rules! function {
 
     // (2-arg) Parse a layout-definition with all arguments.
     (@layout $type:ident | layout($this:ident, $ctx:pat) $code:block) => {
-        impl $crate::func::LayoutFunc for $type {
-            fn layout(&$this, $ctx: LayoutContext) -> LayoutResult<Commands> {
-                Ok($code)
+        impl LayoutFunc for $type {
+            fn layout<'a, 'life0, 'life1, 'async_trait>(
+                &'a $this,
+                $ctx: LayoutContext<'life0, 'life1>
+            ) -> std::pin::Pin<Box<
+                dyn std::future::Future<Output = LayoutResult<Commands<'a>>> + 'async_trait
+            >>
+            where
+                'a: 'async_trait,
+                'life0: 'async_trait,
+                'life1: 'async_trait,
+                Self: 'async_trait,
+            {
+                Box::pin(async move { Ok($code) })
             }
         }
     };

src/func/mod.rs

@@ -4,6 +4,7 @@ use std::any::Any;
 use std::collections::HashMap;
 use std::fmt::{self, Debug, Formatter};
 
+use async_trait::async_trait;
 use self::prelude::*;
 
 #[macro_use]
@@ -24,6 +25,7 @@ pub mod prelude {
     pub use Command::*;
 }
 
+
 /// Types representing functions that are parsed from source code.
 pub trait ParseFunc {
     type Meta: Clone;
@@ -43,12 +45,13 @@ pub trait ParseFunc {
 /// The trait `[LayoutFuncBounds]` is automatically implemented for types which
 /// can be used as functions, that is, all types which fulfill the bounds `Debug
 /// + PartialEq + 'static`.
+#[async_trait(?Send)]
 pub trait LayoutFunc: LayoutFuncBounds {
     /// Layout this function in a given context.
     ///
     /// Returns a sequence of layouting commands which describe what the
     /// function is doing.
-    fn layout(&self, ctx: LayoutContext) -> LayoutResult<Commands>;
+    async fn layout<'a>(&'a self, ctx: LayoutContext<'_, '_>) -> LayoutResult<Commands<'a>>;
 }
 
 impl dyn LayoutFunc {

src/layout/text.rs

@@ -20,8 +20,8 @@ pub struct TextContext<'a, 'p> {
 ///
 /// There is no complex layout involved. The text is simply laid out left-
 /// to-right using the correct font for each character.
-pub fn layout_text(text: &str, ctx: TextContext) -> LayoutResult<Layout> {
-    TextLayouter::new(text, ctx).layout()
+pub async fn layout_text(text: &str, ctx: TextContext<'_, '_>) -> LayoutResult<Layout> {
+    TextLayouter::new(text, ctx).layout().await
 }
 
 /// Layouts text into boxes.
@@ -48,14 +48,14 @@ impl<'a, 'p> TextLayouter<'a, 'p> {
     }
 
     /// Layout the text
-    fn layout(mut self) -> LayoutResult<Layout> {
+    async fn layout(mut self) -> LayoutResult<Layout> {
         if self.ctx.axes.primary.is_positive() {
             for c in self.text.chars() {
-                self.layout_char(c)?;
+                self.layout_char(c).await?;
             }
         } else {
             for c in self.text.chars().rev() {
-                self.layout_char(c)?;
+                self.layout_char(c).await?;
             }
         }
 
@@ -71,8 +71,8 @@ impl<'a, 'p> TextLayouter<'a, 'p> {
     }
 
     /// Layout an individual character.
-    fn layout_char(&mut self, c: char) -> LayoutResult<()> {
-        let (index, char_width) = self.select_font(c)?;
+    async fn layout_char(&mut self, c: char) -> LayoutResult<()> {
+        let (index, char_width) = self.select_font(c).await?;
 
         self.width += char_width;
 
@@ -93,7 +93,7 @@ impl<'a, 'p> TextLayouter<'a, 'p> {
 
     /// Select the best font for a character and return its index along with
     /// the width of the char in the font.
-    fn select_font(&mut self, c: char) -> LayoutResult<(FontIndex, Size)> {
+    async fn select_font(&mut self, c: char) -> LayoutResult<(FontIndex, Size)> {
         let mut loader = self.ctx.loader.borrow_mut();
 
         let query = FontQuery {
@@ -102,7 +102,7 @@ impl<'a, 'p> TextLayouter<'a, 'p> {
             c,
         };
 
-        if let Some((font, index)) = loader.get(query) {
+        if let Some((font, index)) = loader.get(query).await {
             let font_unit_ratio = 1.0 / (font.read_table::<Header>()?.units_per_em as f32);
             let font_unit_to_size = |x| Size::pt(font_unit_ratio * x);
 

src/layout/tree.rs

@@ -1,3 +1,5 @@
+use std::pin::Pin;
+use std::future::Future;
 use smallvec::smallvec;
 
 use crate::func::Command;
@@ -5,10 +7,13 @@ use crate::syntax::{SyntaxTree, Node, FuncCall};
 use crate::style::TextStyle;
 use super::*;
 
+
+type RecursiveResult<'a, T> = Pin<Box<dyn Future<Output=LayoutResult<T>> + 'a>>;
+
 /// Layout a syntax tree into a multibox.
-pub fn layout(tree: &SyntaxTree, ctx: LayoutContext) -> LayoutResult<MultiLayout> {
+pub async fn layout(tree: &SyntaxTree, ctx: LayoutContext<'_, '_>) -> LayoutResult<MultiLayout> {
     let mut layouter = TreeLayouter::new(ctx);
-    layouter.layout(tree)?;
+    layouter.layout(tree).await?;
     layouter.finish()
 }
 
@@ -36,42 +41,44 @@ impl<'a, 'p> TreeLayouter<'a, 'p> {
         }
     }
 
-    fn layout(&mut self, tree: &SyntaxTree) -> LayoutResult<()> {
-        for node in &tree.nodes {
-            match &node.v {
-                Node::Text(text) => self.layout_text(text)?,
+    fn layout<'b>(&'b mut self, tree: &'b SyntaxTree) -> RecursiveResult<'b, ()> {
+        Box::pin(async move {
+            for node in &tree.nodes {
+                match &node.v {
+                    Node::Text(text) => self.layout_text(text).await?,
 
-                Node::Space => self.layout_space(),
-                Node::Newline => self.layout_paragraph()?,
+                    Node::Space => self.layout_space(),
+                    Node::Newline => self.layout_paragraph()?,
 
-                Node::ToggleItalics => self.style.text.variant.style.toggle(),
-                Node::ToggleBolder => {
-                    self.style.text.variant.weight.0 += 300 *
-                        if self.style.text.bolder { -1 } else { 1 };
-                    self.style.text.bolder = !self.style.text.bolder;
-                }
-                Node::ToggleMonospace => {
-                    let list = &mut self.style.text.fallback.list;
-                    match list.get(0).map(|s| s.as_str()) {
-                        Some("monospace") => { list.remove(0); },
-                        _ => list.insert(0, "monospace".to_string()),
+                    Node::ToggleItalics => self.style.text.variant.style.toggle(),
+                    Node::ToggleBolder => {
+                        self.style.text.variant.weight.0 += 300 *
+                            if self.style.text.bolder { -1 } else { 1 };
+                        self.style.text.bolder = !self.style.text.bolder;
                     }
+                    Node::ToggleMonospace => {
+                        let list = &mut self.style.text.fallback.list;
+                        match list.get(0).map(|s| s.as_str()) {
+                            Some("monospace") => { list.remove(0); },
+                            _ => list.insert(0, "monospace".to_string()),
+                        }
+                    }
+
+                    Node::Func(func) => self.layout_func(func).await?,
                 }
-
-                Node::Func(func) => self.layout_func(func)?,
             }
-        }
 
-        Ok(())
+            Ok(())
+        })
     }
 
-    fn layout_text(&mut self, text: &str) -> LayoutResult<()> {
+    async fn layout_text(&mut self, text: &str) -> LayoutResult<()> {
         let layout = layout_text(text, TextContext {
             loader: &self.ctx.loader,
             style: &self.style.text,
             axes: self.ctx.axes,
             alignment: self.ctx.alignment,
-        })?;
+        }).await?;
 
         self.layouter.add(layout)
     }
@@ -84,75 +91,71 @@ impl<'a, 'p> TreeLayouter<'a, 'p> {
         self.layouter.add_secondary_spacing(self.style.text.paragraph_spacing(), PARAGRAPH_KIND)
     }
 
-    fn layout_func(&mut self, func: &FuncCall) -> LayoutResult<()> {
-        let commands = func.0.layout(LayoutContext {
-            style: &self.style,
-            spaces: self.layouter.remaining(),
-            nested: true,
-            debug: false,
-            .. self.ctx
-        })?;
+    fn layout_func<'b>(&'b mut self, func: &'b FuncCall) -> RecursiveResult<'b, ()> {
+        Box::pin(async move {
+            let commands = func.0.layout(LayoutContext {
+                style: &self.style,
+                spaces: self.layouter.remaining(),
+                nested: true,
+                debug: false,
+                .. self.ctx
+            }).await?;
 
-        for command in commands {
-            self.execute(command)?;
-        }
+            for command in commands {
+                use Command::*;
 
-        Ok(())
-    }
+                match command {
+                    LayoutTree(tree) => self.layout(tree).await?,
 
-    fn execute(&mut self, command: Command) -> LayoutResult<()> {
-        use Command::*;
-
-        match command {
-            LayoutTree(tree) => self.layout(tree)?,
-
-            Add(layout) => self.layouter.add(layout)?,
-            AddMultiple(layouts) => self.layouter.add_multiple(layouts)?,
-            SpacingFunc(space, kind, axis) => match axis {
-                Primary => self.layouter.add_primary_spacing(space, kind),
-                Secondary => self.layouter.add_secondary_spacing(space, kind)?,
-            }
-
-            FinishLine => self.layouter.finish_line()?,
-            FinishSpace => self.layouter.finish_space(true)?,
-            BreakParagraph => self.layout_paragraph()?,
-            BreakPage => {
-                if self.ctx.nested {
-                    error!("page break cannot be issued from nested context");
-                }
-
-                self.layouter.finish_space(true)?
-            }
-
-            SetTextStyle(style) => {
-                self.layouter.set_line_spacing(style.line_spacing());
-                self.style.text = style;
-            }
-            SetPageStyle(style) => {
-                if self.ctx.nested {
-                    error!("page style cannot be altered in nested context");
-                }
-
-                self.style.page = style;
-
-                let margins = style.margins();
-                self.ctx.base = style.dimensions.unpadded(margins);
-                self.layouter.set_spaces(smallvec![
-                    LayoutSpace {
-                        dimensions: style.dimensions,
-                        padding: margins,
-                        expansion: LayoutExpansion::new(true, true),
+                    Add(layout) => self.layouter.add(layout)?,
+                    AddMultiple(layouts) => self.layouter.add_multiple(layouts)?,
+                    SpacingFunc(space, kind, axis) => match axis {
+                        Primary => self.layouter.add_primary_spacing(space, kind),
+                        Secondary => self.layouter.add_secondary_spacing(space, kind)?,
                     }
-                ], true);
-            }
-            SetAlignment(alignment) => self.ctx.alignment = alignment,
-            SetAxes(axes) => {
-                self.layouter.set_axes(axes);
-                self.ctx.axes = axes;
-            }
-        }
 
-        Ok(())
+                    FinishLine => self.layouter.finish_line()?,
+                    FinishSpace => self.layouter.finish_space(true)?,
+                    BreakParagraph => self.layout_paragraph()?,
+                    BreakPage => {
+                        if self.ctx.nested {
+                            error!("page break cannot be issued from nested context");
+                        }
+
+                        self.layouter.finish_space(true)?
+                    }
+
+                    SetTextStyle(style) => {
+                        self.layouter.set_line_spacing(style.line_spacing());
+                        self.style.text = style;
+                    }
+                    SetPageStyle(style) => {
+                        if self.ctx.nested {
+                            error!("page style cannot be altered in nested context");
+                        }
+
+                        self.style.page = style;
+
+                        let margins = style.margins();
+                        self.ctx.base = style.dimensions.unpadded(margins);
+                        self.layouter.set_spaces(smallvec![
+                            LayoutSpace {
+                                dimensions: style.dimensions,
+                                padding: margins,
+                                expansion: LayoutExpansion::new(true, true),
+                            }
+                        ], true);
+                    }
+                    SetAlignment(alignment) => self.ctx.alignment = alignment,
+                    SetAxes(axes) => {
+                        self.layouter.set_axes(axes);
+                        self.ctx.axes = axes;
+                    }
+                }
+            }
+
+            Ok(())
+        })
     }
 
     fn finish(self) -> LayoutResult<MultiLayout> {

src/lib.rs

@@ -89,7 +89,7 @@ impl<'p> Typesetter<'p> {
     }
 
     /// Layout a syntax tree and return the produced layout.
-    pub fn layout(&self, tree: &SyntaxTree) -> LayoutResult<MultiLayout> {
+    pub async fn layout(&self, tree: &SyntaxTree) -> LayoutResult<MultiLayout> {
         use crate::layout::prelude::*;
         let margins = self.style.page.margins();
         Ok(layout(
@@ -109,13 +109,13 @@ impl<'p> Typesetter<'p> {
                 nested: false,
                 debug: false,
             },
-        )?)
+        ).await?)
     }
 
     /// Process source code directly into a layout.
-    pub fn typeset(&self, src: &str) -> TypesetResult<MultiLayout> {
+    pub async fn typeset(&self, src: &str) -> TypesetResult<MultiLayout> {
         let tree = self.parse(src)?;
-        let layout = self.layout(&tree)?;
+        let layout = self.layout(&tree).await?;
         Ok(layout)
     }
 }

src/library/align.rs

@@ -28,7 +28,7 @@ function! {
         }
 
         match &self.body {
-            Some(body) => vec![AddMultiple(layout(&body, ctx)?)],
+            Some(body) => vec![AddMultiple(layout(&body, ctx).await?)],
             None => vec![SetAlignment(ctx.alignment)],
         }
     }

src/library/boxed.rs

@@ -46,7 +46,7 @@ function! {
 
             ctx.spaces = smallvec![space];
 
-            match layout(&self.body, ctx) {
+            match layout(&self.body, ctx).await {
                 Ok(layouts) => return Ok(vec![AddMultiple(layouts)]),
                 Err(err) => error = Some(err),
             }

src/library/direction.rs

@@ -36,7 +36,7 @@ function! {
         }
 
         match &self.body {
-            Some(body) => vec![AddMultiple(layout(&body, ctx)?)],
+            Some(body) => vec![AddMultiple(layout(&body, ctx).await?)],
             None => vec![Command::SetAxes(ctx.axes)],
         }
     }

src/library/mod.rs

@@ -55,19 +55,25 @@ function! {
     #[derive(Debug, PartialEq)]
     pub struct FontFamilyFunc {
         body: Option<SyntaxTree>,
-        family: String,
+        list: Vec<String>,
     }
 
     parse(args, body, ctx, meta) {
         FontFamilyFunc {
             body: parse!(optional: body, ctx),
-            family: args.get_pos::<String>()?,
+            list: {
+                args.pos().map(|arg| match arg.v {
+                    Expression::Str(s) |
+                    Expression::Ident(Ident(s)) => Ok(s.to_lowercase()),
+                    _ => error!("expected identifier or string"),
+                }).collect::<LayoutResult<Vec<_>>>()?
+            }
         }
     }
 
     layout(self, ctx) {
         let mut style = ctx.style.text.clone();
-        style.fallback.list = vec![self.family.clone()];
+        style.fallback.list = self.list.clone();
         styled(&self.body, &ctx, style)
     }
 }

src/style.rs

@@ -58,7 +58,7 @@ impl TextStyle {
 }
 
 macro_rules! fallback {
-    (($($f:expr),*), $($c:expr => ($($cf:expr),*)),*) => ({
+    (($($f:expr),*), $($c:expr => ($($cf:expr),*),)*) => ({
         let mut fallback = FontFallbackTree::new(vec![$($f.to_string()),*]);
         $(
             fallback.set_class_list($c.to_string(), vec![$($cf.to_string()),*])
@@ -74,10 +74,11 @@ impl Default for TextStyle {
         TextStyle {
             fallback: fallback! {
                 ("sans-serif"),
-                "serif" => ("source serif pro", "noto serif", "noto emoji"),
-                "sans-serif" => ("source sans pro", "noto sans", "noto emoji"),
-                "monospace" => ("source code pro", "noto sans mono", "noto emoji"),
-                "math" => ("latin modern math", "serif")
+                "serif" => ("source serif pro", "noto serif", "__base"),
+                "sans-serif" => ("source sans pro", "noto sans", "__base"),
+                "monospace" => ("source code pro", "noto sans mono", "__base"),
+                "math" => ("latin modern math", "serif", "__base"),
+                "__base" => ("latin modern math", "noto emoji"),
             },
             variant: FontVariant {
                 style: FontStyle::Normal,

src/syntax/tokens.rs

@@ -189,9 +189,11 @@ impl<'s> Iterator for Tokens<'s> {
             // A string value.
             '"' if self.state == TS::Function => {
                 let start = self.string_index();
+                let mut end = start;
                 let mut escaped = false;
 
-                while let Some((_, c)) = self.chars.next() {
+                while let Some((index, c)) = self.chars.next() {
+                    end = index;
                     if c == '"' && !escaped {
                         break;
                     }
@@ -199,7 +201,6 @@ impl<'s> Iterator for Tokens<'s> {
                     escaped = c == '\\';
                 }
 
-                let end = self.string_index() - 1;
                 Token::Quoted(&self.src[start..end])
             }
 

tests/layout.rs

@@ -6,6 +6,8 @@ use std::io::{BufWriter, Write};
 use std::panic;
 use std::process::Command;
 
+use futures_executor::block_on;
+
 use typstc::Typesetter;
 use typstc::layout::{MultiLayout, Serialize};
 use typstc::size::{Size, Size2D};
@@ -125,7 +127,7 @@ fn compile(typesetter: &Typesetter, src: &str) -> Option<MultiLayout> {
 
         // Warmup.
         let warmup_start = Instant::now();
-        let is_ok = typesetter.typeset(&src).is_ok();
+        let is_ok = block_on(typesetter.typeset(&src)).is_ok();
         let warmup_end = Instant::now();
 
         // Only continue if the typesetting was successful.
@@ -133,7 +135,7 @@ fn compile(typesetter: &Typesetter, src: &str) -> Option<MultiLayout> {
             let start = Instant::now();
             let tree = typesetter.parse(&src).unwrap();
             let mid = Instant::now();
-            typesetter.layout(&tree).unwrap();
+            block_on(typesetter.layout(&tree)).unwrap();
             let end = Instant::now();
 
             println!(" - cold start:  {:?}", warmup_end - warmup_start);
@@ -144,7 +146,7 @@ fn compile(typesetter: &Typesetter, src: &str) -> Option<MultiLayout> {
         }
     };
 
-    match typesetter.typeset(&src) {
+    match block_on(typesetter.typeset(&src)) {
         Ok(layouts) => Some(layouts),
         Err(err) => {
             println!(" - compilation failed: {}", err);