Asyncify font loading 🪐

Cargo.toml

@@ -6,20 +6,28 @@ edition = "2018"
 build = "build.rs"
 
 [dependencies]
+toddle = { path = "../toddle", default-features = false }
 tide = { path = "../tide" }
-toddle = { path = "../toddle" }
 byteorder = "1"
 smallvec = "0.6.10"
 unicode-xid = "0.1.0"
+async-trait = "0.1.22"
+futures-executor = { version = "0.3", optional = true }
+
+[features]
+default = ["fs-provider", "futures-executor"]
+fs-provider = ["toddle/fs-provider"]
 
 [[bin]]
 name = "typst-bin"
 path = "src/bin/main.rs"
+required-features = ["futures-executor"]
 
 [[test]]
 name = "layout"
 path = "tests/layout.rs"
 harness = false
+required-features = ["futures-executor"]
 
 [[test]]
 name = "parse"

src/bin/main.rs

@@ -2,6 +2,8 @@ use std::fs::{File, read_to_string};
 use std::io::BufWriter;
 use std::path::{Path, PathBuf};
 
+use futures_executor::block_on;
+
 use typstc::Typesetter;
 use typstc::toddle::query::FileSystemFontProvider;
 use typstc::export::pdf::PdfExporter;
@@ -39,7 +41,7 @@ fn run() -> Result<(), Box<dyn std::error::Error>> {
     let provider = FileSystemFontProvider::from_index("../fonts/index.json").unwrap();
     typesetter.add_font_provider(provider);
 
-    let layouts = typesetter.typeset(&src)?;
+    let layouts = block_on(typesetter.typeset(&src))?;
 
     let exporter = PdfExporter::new();
     let writer = BufWriter::new(File::create(&dest)?);

src/func/macros.rs

@@ -108,9 +108,20 @@ macro_rules! function {
 
     // (2-arg) Parse a layout-definition with all arguments.
     (@layout $type:ident | layout($this:ident, $ctx:pat) $code:block) => {
-        impl $crate::func::LayoutFunc for $type {
+        impl LayoutFunc for $type {
-            fn layout(&$this, $ctx: LayoutContext) -> LayoutResult<Commands> {
+            fn layout<'a, 'life0, 'life1, 'async_trait>(
-                Ok($code)
+                &'a $this,
+                $ctx: LayoutContext<'life0, 'life1>
+            ) -> std::pin::Pin<Box<
+                dyn std::future::Future<Output = LayoutResult<Commands<'a>>> + 'async_trait
+            >>
+            where
+                'a: 'async_trait,
+                'life0: 'async_trait,
+                'life1: 'async_trait,
+                Self: 'async_trait,
+            {
+                Box::pin(async move { Ok($code) })
             }
         }
     };

src/func/mod.rs

@@ -4,6 +4,7 @@ use std::any::Any;
 use std::collections::HashMap;
 use std::fmt::{self, Debug, Formatter};
 
+use async_trait::async_trait;
 use self::prelude::*;
 
 #[macro_use]
@@ -24,6 +25,7 @@ pub mod prelude {
     pub use Command::*;
 }
 
+
 /// Types representing functions that are parsed from source code.
 pub trait ParseFunc {
     type Meta: Clone;
@@ -43,12 +45,13 @@ pub trait ParseFunc {
 /// The trait `[LayoutFuncBounds]` is automatically implemented for types which
 /// can be used as functions, that is, all types which fulfill the bounds `Debug
 /// + PartialEq + 'static`.
+#[async_trait(?Send)]
 pub trait LayoutFunc: LayoutFuncBounds {
     /// Layout this function in a given context.
     ///
     /// Returns a sequence of layouting commands which describe what the
     /// function is doing.
-    fn layout(&self, ctx: LayoutContext) -> LayoutResult<Commands>;
+    async fn layout<'a>(&'a self, ctx: LayoutContext<'_, '_>) -> LayoutResult<Commands<'a>>;
 }
 
 impl dyn LayoutFunc {

src/layout/text.rs

@@ -20,8 +20,8 @@ pub struct TextContext<'a, 'p> {
 ///
 /// There is no complex layout involved. The text is simply laid out left-
 /// to-right using the correct font for each character.
-pub fn layout_text(text: &str, ctx: TextContext) -> LayoutResult<Layout> {
+pub async fn layout_text(text: &str, ctx: TextContext<'_, '_>) -> LayoutResult<Layout> {
-    TextLayouter::new(text, ctx).layout()
+    TextLayouter::new(text, ctx).layout().await
 }
 
 /// Layouts text into boxes.
@@ -48,14 +48,14 @@ impl<'a, 'p> TextLayouter<'a, 'p> {
     }
 
     /// Layout the text
-    fn layout(mut self) -> LayoutResult<Layout> {
+    async fn layout(mut self) -> LayoutResult<Layout> {
         if self.ctx.axes.primary.is_positive() {
             for c in self.text.chars() {
-                self.layout_char(c)?;
+                self.layout_char(c).await?;
             }
         } else {
             for c in self.text.chars().rev() {
-                self.layout_char(c)?;
+                self.layout_char(c).await?;
             }
         }
 
@@ -71,8 +71,8 @@ impl<'a, 'p> TextLayouter<'a, 'p> {
     }
 
     /// Layout an individual character.
-    fn layout_char(&mut self, c: char) -> LayoutResult<()> {
+    async fn layout_char(&mut self, c: char) -> LayoutResult<()> {
-        let (index, char_width) = self.select_font(c)?;
+        let (index, char_width) = self.select_font(c).await?;
 
         self.width += char_width;
 
@@ -93,7 +93,7 @@ impl<'a, 'p> TextLayouter<'a, 'p> {
 
     /// Select the best font for a character and return its index along with
     /// the width of the char in the font.
-    fn select_font(&mut self, c: char) -> LayoutResult<(FontIndex, Size)> {
+    async fn select_font(&mut self, c: char) -> LayoutResult<(FontIndex, Size)> {
         let mut loader = self.ctx.loader.borrow_mut();
 
         let query = FontQuery {
@@ -102,7 +102,7 @@ impl<'a, 'p> TextLayouter<'a, 'p> {
             c,
         };
 
-        if let Some((font, index)) = loader.get(query) {
+        if let Some((font, index)) = loader.get(query).await {
             let font_unit_ratio = 1.0 / (font.read_table::<Header>()?.units_per_em as f32);
             let font_unit_to_size = |x| Size::pt(font_unit_ratio * x);
 

src/layout/tree.rs

@@ -1,3 +1,5 @@
+use std::pin::Pin;
+use std::future::Future;
 use smallvec::smallvec;
 
 use crate::func::Command;
@@ -5,10 +7,13 @@ use crate::syntax::{SyntaxTree, Node, FuncCall};
 use crate::style::TextStyle;
 use super::*;
 
+
+type RecursiveResult<'a, T> = Pin<Box<dyn Future<Output=LayoutResult<T>> + 'a>>;
+
 /// Layout a syntax tree into a multibox.
-pub fn layout(tree: &SyntaxTree, ctx: LayoutContext) -> LayoutResult<MultiLayout> {
+pub async fn layout(tree: &SyntaxTree, ctx: LayoutContext<'_, '_>) -> LayoutResult<MultiLayout> {
     let mut layouter = TreeLayouter::new(ctx);
-    layouter.layout(tree)?;
+    layouter.layout(tree).await?;
     layouter.finish()
 }
 
@@ -36,10 +41,11 @@ impl<'a, 'p> TreeLayouter<'a, 'p> {
         }
     }
 
-    fn layout(&mut self, tree: &SyntaxTree) -> LayoutResult<()> {
+    fn layout<'b>(&'b mut self, tree: &'b SyntaxTree) -> RecursiveResult<'b, ()> {
+        Box::pin(async move {
             for node in &tree.nodes {
                 match &node.v {
-                Node::Text(text) => self.layout_text(text)?,
+                    Node::Text(text) => self.layout_text(text).await?,
 
                     Node::Space => self.layout_space(),
                     Node::Newline => self.layout_paragraph()?,
@@ -58,20 +64,21 @@ impl<'a, 'p> TreeLayouter<'a, 'p> {
                         }
                     }
 
-                Node::Func(func) => self.layout_func(func)?,
+                    Node::Func(func) => self.layout_func(func).await?,
                 }
             }
 
             Ok(())
+        })
     }
 
-    fn layout_text(&mut self, text: &str) -> LayoutResult<()> {
+    async fn layout_text(&mut self, text: &str) -> LayoutResult<()> {
         let layout = layout_text(text, TextContext {
             loader: &self.ctx.loader,
             style: &self.style.text,
             axes: self.ctx.axes,
             alignment: self.ctx.alignment,
-        })?;
+        }).await?;
 
         self.layouter.add(layout)
     }
@@ -84,27 +91,21 @@ impl<'a, 'p> TreeLayouter<'a, 'p> {
         self.layouter.add_secondary_spacing(self.style.text.paragraph_spacing(), PARAGRAPH_KIND)
     }
 
-    fn layout_func(&mut self, func: &FuncCall) -> LayoutResult<()> {
+    fn layout_func<'b>(&'b mut self, func: &'b FuncCall) -> RecursiveResult<'b, ()> {
+        Box::pin(async move {
             let commands = func.0.layout(LayoutContext {
                 style: &self.style,
                 spaces: self.layouter.remaining(),
                 nested: true,
                 debug: false,
                 .. self.ctx
-        })?;
+            }).await?;
 
             for command in commands {
-            self.execute(command)?;
-        }
-
-        Ok(())
-    }
-
-    fn execute(&mut self, command: Command) -> LayoutResult<()> {
                 use Command::*;
 
                 match command {
-            LayoutTree(tree) => self.layout(tree)?,
+                    LayoutTree(tree) => self.layout(tree).await?,
 
                     Add(layout) => self.layouter.add(layout)?,
                     AddMultiple(layouts) => self.layouter.add_multiple(layouts)?,
@@ -151,8 +152,10 @@ impl<'a, 'p> TreeLayouter<'a, 'p> {
                         self.ctx.axes = axes;
                     }
                 }
+            }
 
             Ok(())
+        })
     }
 
     fn finish(self) -> LayoutResult<MultiLayout> {

src/lib.rs

@@ -89,7 +89,7 @@ impl<'p> Typesetter<'p> {
     }
 
     /// Layout a syntax tree and return the produced layout.
-    pub fn layout(&self, tree: &SyntaxTree) -> LayoutResult<MultiLayout> {
+    pub async fn layout(&self, tree: &SyntaxTree) -> LayoutResult<MultiLayout> {
         use crate::layout::prelude::*;
         let margins = self.style.page.margins();
         Ok(layout(
@@ -109,13 +109,13 @@ impl<'p> Typesetter<'p> {
                 nested: false,
                 debug: false,
             },
-        )?)
+        ).await?)
     }
 
     /// Process source code directly into a layout.
-    pub fn typeset(&self, src: &str) -> TypesetResult<MultiLayout> {
+    pub async fn typeset(&self, src: &str) -> TypesetResult<MultiLayout> {
         let tree = self.parse(src)?;
-        let layout = self.layout(&tree)?;
+        let layout = self.layout(&tree).await?;
         Ok(layout)
     }
 }

src/library/align.rs

@@ -28,7 +28,7 @@ function! {
         }
 
         match &self.body {
-            Some(body) => vec![AddMultiple(layout(&body, ctx)?)],
+            Some(body) => vec![AddMultiple(layout(&body, ctx).await?)],
             None => vec![SetAlignment(ctx.alignment)],
         }
     }

src/library/boxed.rs

@@ -46,7 +46,7 @@ function! {
 
             ctx.spaces = smallvec![space];
 
-            match layout(&self.body, ctx) {
+            match layout(&self.body, ctx).await {
                 Ok(layouts) => return Ok(vec![AddMultiple(layouts)]),
                 Err(err) => error = Some(err),
             }

src/library/direction.rs

@@ -36,7 +36,7 @@ function! {
         }
 
         match &self.body {
-            Some(body) => vec![AddMultiple(layout(&body, ctx)?)],
+            Some(body) => vec![AddMultiple(layout(&body, ctx).await?)],
             None => vec![Command::SetAxes(ctx.axes)],
         }
     }

src/library/mod.rs

@@ -55,19 +55,25 @@ function! {
     #[derive(Debug, PartialEq)]
     pub struct FontFamilyFunc {
         body: Option<SyntaxTree>,
-        family: String,
+        list: Vec<String>,
     }
 
     parse(args, body, ctx, meta) {
         FontFamilyFunc {
             body: parse!(optional: body, ctx),
-            family: args.get_pos::<String>()?,
+            list: {
+                args.pos().map(|arg| match arg.v {
+                    Expression::Str(s) |
+                    Expression::Ident(Ident(s)) => Ok(s.to_lowercase()),
+                    _ => error!("expected identifier or string"),
+                }).collect::<LayoutResult<Vec<_>>>()?
+            }
         }
     }
 
     layout(self, ctx) {
         let mut style = ctx.style.text.clone();
-        style.fallback.list = vec![self.family.clone()];
+        style.fallback.list = self.list.clone();
         styled(&self.body, &ctx, style)
     }
 }

src/style.rs

@@ -58,7 +58,7 @@ impl TextStyle {
 }
 
 macro_rules! fallback {
-    (($($f:expr),*), $($c:expr => ($($cf:expr),*)),*) => ({
+    (($($f:expr),*), $($c:expr => ($($cf:expr),*),)*) => ({
         let mut fallback = FontFallbackTree::new(vec![$($f.to_string()),*]);
         $(
             fallback.set_class_list($c.to_string(), vec![$($cf.to_string()),*])
@@ -74,10 +74,11 @@ impl Default for TextStyle {
         TextStyle {
             fallback: fallback! {
                 ("sans-serif"),
-                "serif" => ("source serif pro", "noto serif", "noto emoji"),
+                "serif" => ("source serif pro", "noto serif", "__base"),
-                "sans-serif" => ("source sans pro", "noto sans", "noto emoji"),
+                "sans-serif" => ("source sans pro", "noto sans", "__base"),
-                "monospace" => ("source code pro", "noto sans mono", "noto emoji"),
+                "monospace" => ("source code pro", "noto sans mono", "__base"),
-                "math" => ("latin modern math", "serif")
+                "math" => ("latin modern math", "serif", "__base"),
+                "__base" => ("latin modern math", "noto emoji"),
             },
             variant: FontVariant {
                 style: FontStyle::Normal,

src/syntax/tokens.rs

@@ -189,9 +189,11 @@ impl<'s> Iterator for Tokens<'s> {
             // A string value.
             '"' if self.state == TS::Function => {
                 let start = self.string_index();
+                let mut end = start;
                 let mut escaped = false;
 
-                while let Some((_, c)) = self.chars.next() {
+                while let Some((index, c)) = self.chars.next() {
+                    end = index;
                     if c == '"' && !escaped {
                         break;
                     }
@@ -199,7 +201,6 @@ impl<'s> Iterator for Tokens<'s> {
                     escaped = c == '\\';
                 }
 
-                let end = self.string_index() - 1;
                 Token::Quoted(&self.src[start..end])
             }
 

tests/layout.rs

@@ -6,6 +6,8 @@ use std::io::{BufWriter, Write};
 use std::panic;
 use std::process::Command;
 
+use futures_executor::block_on;
+
 use typstc::Typesetter;
 use typstc::layout::{MultiLayout, Serialize};
 use typstc::size::{Size, Size2D};
@@ -125,7 +127,7 @@ fn compile(typesetter: &Typesetter, src: &str) -> Option<MultiLayout> {
 
         // Warmup.
         let warmup_start = Instant::now();
-        let is_ok = typesetter.typeset(&src).is_ok();
+        let is_ok = block_on(typesetter.typeset(&src)).is_ok();
         let warmup_end = Instant::now();
 
         // Only continue if the typesetting was successful.
@@ -133,7 +135,7 @@ fn compile(typesetter: &Typesetter, src: &str) -> Option<MultiLayout> {
             let start = Instant::now();
             let tree = typesetter.parse(&src).unwrap();
             let mid = Instant::now();
-            typesetter.layout(&tree).unwrap();
+            block_on(typesetter.layout(&tree)).unwrap();
             let end = Instant::now();
 
             println!(" - cold start:  {:?}", warmup_end - warmup_start);
@@ -144,7 +146,7 @@ fn compile(typesetter: &Typesetter, src: &str) -> Option<MultiLayout> {
         }
     };
 
-    match typesetter.typeset(&src) {
+    match block_on(typesetter.typeset(&src)) {
         Ok(layouts) => Some(layouts),
         Err(err) => {
             println!(" - compilation failed: {}", err);