Parallelize tests (#900)

Cargo.lock

@@ -1089,9 +1089,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
 [[package]]
 name = "libc"
-version = "0.2.141"
+version = "0.2.142"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5"
+checksum = "6a987beff54b60ffa6d51982e1aa1146bc42f19bd26be28b0586f252fccf5317"
 
 [[package]]
 name = "libdeflate-sys"
@@ -1704,9 +1704,9 @@ dependencies = [
 
 [[package]]
 name = "rustix"
-version = "0.37.12"
+version = "0.37.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "722529a737f5a942fdbac3a46cee213053196737c5eaa3386d52e85b786f2659"
+checksum = "f79bef90eb6d984c72722595b5b1348ab39275a5e5123faca6863bf07d75a4e0"
 dependencies = [
  "bitflags",
  "errno",
@@ -2373,6 +2373,7 @@ dependencies = [
  "iai",
  "once_cell",
  "oxipng",
+ "rayon",
  "tiny-skia",
  "ttf-parser 0.17.1",
  "typst",

src/syntax/source.rs

@@ -18,6 +18,7 @@ use crate::util::{PathExt, StrExt};
 ///
 /// All line and column indices start at zero, just like byte indices. Only for
 /// user-facing display, you should add 1 to them.
+#[derive(Clone)]
 pub struct Source {
     id: SourceId,
     path: PathBuf,

tests/Cargo.toml

@@ -13,6 +13,7 @@ elsa = "1.7"
 iai = { git = "https://github.com/reknih/iai" }
 once_cell = "1"
 oxipng = "8.0.0"
+rayon = "1.7.0"
 tiny-skia = "0.6.2"
 ttf-parser = "0.17"
 unscanny = "0.1"

tests/src/tests.rs

@@ -3,15 +3,24 @@
 use std::cell::{RefCell, RefMut};
 use std::collections::HashMap;
 use std::ffi::OsStr;
+use std::fmt::Write as FmtWrite;
 use std::fs;
+use std::io::Write;
 use std::ops::Range;
 use std::path::{Path, PathBuf};
+use std::{env, io};
 
+use clap::Parser;
 use comemo::{Prehashed, Track};
 use elsa::FrozenVec;
 use once_cell::unsync::OnceCell;
 use oxipng::{InFile, Options, OutFile};
+use rayon::iter::ParallelBridge;
+use rayon::iter::ParallelIterator;
 use tiny_skia as sk;
+use unscanny::Scanner;
+use walkdir::WalkDir;
+
 use typst::diag::{bail, FileError, FileResult};
 use typst::doc::{Document, Frame, FrameItem, Meta};
 use typst::eval::{func, Library, Value};
@@ -22,8 +31,6 @@ use typst::util::{Buffer, PathExt};
 use typst::World;
 use typst_library::layout::PageElem;
 use typst_library::text::{TextElem, TextSize};
-use unscanny::Scanner;
-use walkdir::WalkDir;
 
 const TYP_DIR: &str = "typ";
 const REF_DIR: &str = "ref";
@@ -32,8 +39,6 @@ const PDF_DIR: &str = "pdf";
 const FONT_DIR: &str = "../assets/fonts";
 const FILE_DIR: &str = "../assets/files";
 
-use clap::Parser;
-
 #[derive(Debug, Clone, Parser)]
 #[clap(name = "typst-test", author)]
 struct Args {
@@ -43,7 +48,7 @@ struct Args {
     subtest: Option<usize>,
     #[arg(long)]
     exact: bool,
-    #[arg(long)]
+    #[arg(long, default_value_t = env::var_os("UPDATE_EXPECT").is_some())]
     update: bool,
     #[arg(long)]
     pdf: bool,
@@ -78,53 +83,49 @@ impl Args {
 
 fn main() {
     let args = Args::parse();
-    let mut filtered = Vec::new();
+
-    // Since different tests can affect each other through the memoization
+    // Create loader and context.
-    // cache, a deterministic order is important for reproducibility.
+    let world = TestWorld::new(args.print);
-    for entry in WalkDir::new("typ").sort_by_file_name() {
+
+    println!("Running tests...");
+    let results = WalkDir::new("typ")
+        .into_iter()
+        .par_bridge()
+        .filter_map(|entry| {
             let entry = entry.unwrap();
             if entry.depth() == 0 {
-            continue;
+                return None;
             }
 
             if entry.path().starts_with("typ/benches") {
-            continue;
+                return None;
             }
 
             let src_path = entry.into_path();
             if src_path.extension() != Some(OsStr::new("typ")) {
-            continue;
+                return None;
             }
 
             if args.matches(&src_path) {
-            filtered.push(src_path);
+                Some(src_path)
+            } else {
+                None
             }
-    }
+        })
-
+        .map_with(world, |world, src_path| {
-    let len = filtered.len();
-    if len == 1 {
-        println!("Running test ...");
-    } else if len > 1 {
-        println!("Running {len} tests");
-    }
-
-    // Create loader and context.
-    let mut world = TestWorld::new(args.print);
-
-    // Run all the tests.
-    let mut ok = 0;
-    for src_path in filtered {
             let path = src_path.strip_prefix(TYP_DIR).unwrap();
             let png_path = Path::new(PNG_DIR).join(path).with_extension("png");
             let ref_path = Path::new(REF_DIR).join(path).with_extension("png");
             let pdf_path =
                 args.pdf.then(|| Path::new(PDF_DIR).join(path).with_extension("pdf"));
 
-        ok +=
+            test(world, &src_path, &png_path, &ref_path, pdf_path.as_deref(), &args)
-            test(&mut world, &src_path, &png_path, &ref_path, pdf_path.as_deref(), &args)
+                as usize
-                as usize;
+        })
-    }
+        .collect::<Vec<_>>();
 
+    let len = results.len();
+    let ok = results.iter().sum::<usize>();
     if len > 1 {
         println!("{ok} / {len} tests passed.");
     }
@@ -158,14 +159,15 @@ fn library() -> Library {
     /// Returns:
     #[func]
     fn print(#[variadic] values: Vec<Value>) -> Value {
-        print!("> ");
+        let mut stdout = io::stdout().lock();
+        write!(stdout, "> ").unwrap();
         for (i, value) in values.into_iter().enumerate() {
             if i > 0 {
-                print!(", ")
+                write!(stdout, ", ").unwrap();
             }
-            print!("{value:?}");
+            write!(stdout, "{value:?}").unwrap();
         }
-        println!();
+        writeln!(stdout).unwrap();
         Value::None
     }
 
@@ -206,7 +208,21 @@ struct TestWorld {
     main: SourceId,
 }
 
-#[derive(Default)]
+impl Clone for TestWorld {
+    fn clone(&self) -> Self {
+        Self {
+            print: self.print,
+            library: self.library.clone(),
+            book: self.book.clone(),
+            fonts: self.fonts.clone(),
+            paths: self.paths.clone(),
+            sources: FrozenVec::from_iter(self.sources.iter().cloned().map(Box::new)),
+            main: self.main,
+        }
+    }
+}
+
+#[derive(Default, Clone)]
 struct PathSlot {
     source: OnceCell<FileResult<SourceId>>,
     buffer: OnceCell<FileResult<Buffer>>,
@@ -222,7 +238,7 @@ impl TestWorld {
             .filter_map(|e| e.ok())
             .filter(|entry| entry.file_type().is_file())
         {
-            let data = std::fs::read(entry.path()).unwrap();
+            let data = fs::read(entry.path()).unwrap();
             fonts.extend(Font::iter(data.into()));
         }
 
@@ -337,10 +353,10 @@ fn test(
     args: &Args,
 ) -> bool {
     let name = src_path.strip_prefix(TYP_DIR).unwrap_or(src_path);
-    println!("Testing {}", name.display());
 
     let text = fs::read_to_string(src_path).unwrap();
 
+    let mut output = String::new();
     let mut ok = true;
     let mut updated = false;
     let mut frames = vec![];
@@ -353,7 +369,7 @@ fn test(
     for (i, &part) in parts.iter().enumerate() {
         if let Some(x) = args.subtest {
             if x != i {
-                println!("skipped subtest {i}");
+                writeln!(output, "  Skipped subtest {i}.").unwrap();
                 continue;
             }
         }
@@ -370,8 +386,16 @@ fn test(
                 }
             }
         } else {
-            let (part_ok, compare_here, part_frames) =
+            let (part_ok, compare_here, part_frames) = test_part(
-                test_part(world, src_path, part.into(), i, compare_ref, line, &mut rng);
+                &mut output,
+                world,
+                src_path,
+                part.into(),
+                i,
+                compare_ref,
+                line,
+                &mut rng,
+            );
             ok &= part_ok;
             compare_ever |= compare_here;
             frames.extend(part_frames);
@@ -390,7 +414,7 @@ fn test(
 
         if world.print.frames {
             for frame in &document.pages {
-                println!("Frame:\n{:#?}\n", frame);
+                writeln!(output, "{:#?}\n", frame).unwrap();
             }
         }
 
@@ -411,7 +435,7 @@ fn test(
                     update_image(png_path, ref_path);
                     updated = true;
                 } else {
-                    println!("  Does not match reference image. ❌");
+                    writeln!(output, "  Does not match reference image.").unwrap();
                     ok = false;
                 }
             }
@@ -420,24 +444,32 @@ fn test(
                 update_image(png_path, ref_path);
                 updated = true;
             } else {
-                println!("  Failed to open reference image. ❌");
+                writeln!(output, "  Failed to open reference image.").unwrap();
                 ok = false;
             }
         }
     }
 
-    if ok && !updated {
+    {
-        if world.print == PrintConfig::default() {
+        let mut stdout = io::stdout().lock();
-            print!("\x1b[1A");
+        stdout.write_all(name.to_string_lossy().as_bytes()).unwrap();
+        if ok {
+            writeln!(stdout, " ✔").unwrap();
+        } else {
+            writeln!(stdout, " ❌").unwrap();
+        }
+        if updated {
+            writeln!(stdout, "  Updated reference image.").unwrap();
+        }
+        if !output.is_empty() {
+            stdout.write_all(output.as_bytes()).unwrap();
         }
-        println!("Testing {} ✔", name.display());
     }
 
     ok
 }
 
 fn update_image(png_path: &Path, ref_path: &Path) {
-    println!("  Updated reference image. ✔");
     oxipng::optimize(
         &InFile::Path(png_path.to_owned()),
         &OutFile::Path(Some(ref_path.to_owned())),
@@ -446,7 +478,9 @@ fn update_image(png_path: &Path, ref_path: &Path) {
     .unwrap();
 }
 
+#[allow(clippy::too_many_arguments)]
 fn test_part(
+    output: &mut String,
     world: &mut TestWorld,
     src_path: &Path,
     text: String,
@@ -460,14 +494,14 @@ fn test_part(
     let id = world.set(src_path, text);
     let source = world.source(id);
     if world.print.syntax {
-        println!("Syntax Tree:\n{:#?}\n", source.root())
+        writeln!(output, "Syntax Tree:\n{:#?}\n", source.root()).unwrap();
     }
 
     let (local_compare_ref, mut ref_errors) = parse_metadata(source);
     let compare_ref = local_compare_ref.unwrap_or(compare_ref);
 
-    ok &= test_spans(source.root());
+    ok &= test_spans(output, source.root());
-    ok &= test_reparse(world.source(id).text(), i, rng);
+    ok &= test_reparse(output, world.source(id).text(), i, rng);
 
     if world.print.model {
         let world = (world as &dyn World).track();
@@ -475,7 +509,7 @@ fn test_part(
         let mut tracer = typst::eval::Tracer::default();
         let module =
             typst::eval::eval(world, route.track(), tracer.track_mut(), source).unwrap();
-        println!("Model:\n{:#?}\n", module.content());
+        writeln!(output, "Model:\n{:#?}\n", module.content()).unwrap();
     }
 
     let (mut frames, errors) = match typst::compile(world) {
@@ -500,21 +534,21 @@ fn test_part(
     ref_errors.sort_by_key(|error| error.0.start);
 
     if errors != ref_errors {
-        println!("  Subtest {i} does not match expected errors. ❌");
+        writeln!(output, "  Subtest {i} does not match expected errors.").unwrap();
         ok = false;
 
         let source = world.source(id);
         for error in errors.iter() {
             if !ref_errors.contains(error) {
-                print!("    Not annotated | ");
+                write!(output, "    Not annotated | ").unwrap();
-                print_error(source, line, error);
+                print_error(output, source, line, error);
             }
         }
 
         for error in ref_errors.iter() {
             if !errors.contains(error) {
-                print!("    Not emitted   | ");
+                write!(output, "    Not emitted   | ").unwrap();
-                print_error(source, line, error);
+                print_error(output, source, line, error);
             }
         }
     }
@@ -551,7 +585,7 @@ fn parse_metadata(source: &Source) -> (Option<bool>, Vec<(Range<usize>, String)>
             source.line_column_to_byte(line, column).unwrap()
         };
 
-        let Some(rest) = line.strip_prefix("// Error: ") else { continue };
+        let Some(rest) = line.strip_prefix("// Error: ") else { continue; };
         let mut s = Scanner::new(rest);
         let start = pos(&mut s);
         let end = if s.eat_if('-') { pos(&mut s) } else { start };
@@ -563,12 +597,18 @@ fn parse_metadata(source: &Source) -> (Option<bool>, Vec<(Range<usize>, String)>
     (compare_ref, errors)
 }
 
-fn print_error(source: &Source, line: usize, (range, message): &(Range<usize>, String)) {
+fn print_error(
+    output: &mut String,
+    source: &Source,
+    line: usize,
+    (range, message): &(Range<usize>, String),
+) {
     let start_line = 1 + line + source.byte_to_line(range.start).unwrap();
     let start_col = 1 + source.byte_to_column(range.start).unwrap();
     let end_line = 1 + line + source.byte_to_line(range.end).unwrap();
     let end_col = 1 + source.byte_to_column(range.end).unwrap();
-    println!("Error: {start_line}:{start_col}-{end_line}:{end_col}: {message}");
+    writeln!(output, "Error: {start_line}:{start_col}-{end_line}:{end_col}: {message}")
+        .unwrap();
 }
 
 /// Pseudorandomly edit the source file and test whether a reparse produces the
@@ -577,7 +617,12 @@ fn print_error(source: &Source, line: usize, (range, message): &(Range<usize>, S
 /// The method will first inject 10 strings once every 400 source characters
 /// and then select 5 leaf node boundaries to inject an additional, randomly
 /// chosen string from the injection list.
-fn test_reparse(text: &str, i: usize, rng: &mut LinearShift) -> bool {
+fn test_reparse(
+    output: &mut String,
+    text: &str,
+    i: usize,
+    rng: &mut LinearShift,
+) -> bool {
     let supplements = [
         "[",
         "]",
@@ -608,7 +653,7 @@ fn test_reparse(text: &str, i: usize, rng: &mut LinearShift) -> bool {
 
     let mut ok = true;
 
-    let apply = |replace: std::ops::Range<usize>, with| {
+    let mut apply = |replace: Range<usize>, with| {
         let mut incr_source = Source::detached(text);
         if incr_source.root().len() != text.len() {
             println!(
@@ -627,7 +672,7 @@ fn test_reparse(text: &str, i: usize, rng: &mut LinearShift) -> bool {
         let mut incr_root = incr_source.root().clone();
 
         // Ensures that the span numbering invariants hold.
-        let spans_ok = test_spans(&ref_root) && test_spans(&incr_root);
+        let spans_ok = test_spans(output, &ref_root) && test_spans(output, &incr_root);
 
         // Remove all spans so that the comparison works out.
         let tree_ok = {
@@ -637,13 +682,19 @@ fn test_reparse(text: &str, i: usize, rng: &mut LinearShift) -> bool {
         };
 
         if !tree_ok {
-            println!(
+            writeln!(
+                output,
                 "    Subtest {i} reparse differs from clean parse when inserting '{with}' at {}-{} ❌\n",
                 replace.start, replace.end,
-            );
+            ).unwrap();
-            println!("    Expected reference tree:\n{ref_root:#?}\n");
+            writeln!(output, "    Expected reference tree:\n{ref_root:#?}\n").unwrap();
-            println!("    Found incremental tree:\n{incr_root:#?}");
+            writeln!(output, "    Found incremental tree:\n{incr_root:#?}").unwrap();
-            println!("    Full source ({}):\n\"{edited_src:?}\"", edited_src.len());
+            writeln!(
+                output,
+                "    Full source ({}):\n\"{edited_src:?}\"",
+                edited_src.len()
+            )
+            .unwrap();
         }
 
         spans_ok && tree_ok
@@ -687,22 +738,27 @@ fn leafs(node: &SyntaxNode) -> Vec<SyntaxNode> {
 
 /// Ensure that all spans are properly ordered (and therefore unique).
 #[track_caller]
-fn test_spans(root: &SyntaxNode) -> bool {
+fn test_spans(output: &mut String, root: &SyntaxNode) -> bool {
-    test_spans_impl(root, 0..u64::MAX)
+    test_spans_impl(output, root, 0..u64::MAX)
 }
 
 #[track_caller]
-fn test_spans_impl(node: &SyntaxNode, within: Range<u64>) -> bool {
+fn test_spans_impl(output: &mut String, node: &SyntaxNode, within: Range<u64>) -> bool {
     if !within.contains(&node.span().number()) {
-        eprintln!("    Node: {node:#?}");
+        writeln!(output, "    Node: {node:#?}").unwrap();
-        eprintln!("    Wrong span order: {} not in {within:?} ❌", node.span().number(),);
+        writeln!(
+            output,
+            "    Wrong span order: {} not in {within:?} ❌",
+            node.span().number()
+        )
+        .unwrap();
     }
 
     let start = node.span().number() + 1;
     let mut children = node.children().peekable();
     while let Some(child) = children.next() {
         let end = children.peek().map_or(within.end, |next| next.span().number());
-        if !test_spans_impl(child, start..end) {
+        if !test_spans_impl(output, child, start..end) {
             return false;
         }
     }