Better test runner (#3922)

.gitignore

@@ -6,11 +6,8 @@ desktop.ini
 .DS_Store
 
 # Tests and benchmarks
-tests/png
-tests/pdf
-tests/svg
-tests/target
-tests/typ/**/*.pdf
+tests/store
+tests/suite/**/*.pdf
 tests/fuzz/target
 tests/fuzz/corpus
 tests/fuzz/artifacts
@@ -23,6 +20,7 @@ tarpaulin-report.html
 
 # Node
 node_modules
+tools/test-helper/dist
 package-lock.json
 
 # Nix

Cargo.lock

@@ -2675,8 +2675,11 @@ dependencies = [
  "ecow",
  "if_chain",
  "log",
+ "once_cell",
  "serde",
  "typst",
+ "typst-assets",
+ "typst-dev-assets",
  "unscanny",
 ]
 
@@ -2773,13 +2776,13 @@ dependencies = [
  "ecow",
  "once_cell",
  "oxipng",
+ "parking_lot",
  "rayon",
  "tiny-skia",
  "ttf-parser",
  "typst",
  "typst-assets",
  "typst-dev-assets",
- "typst-ide",
  "typst-pdf",
  "typst-render",
  "typst-svg",

crates/typst-cli/src/world.rs

@@ -226,7 +226,7 @@ struct FileSlot {
 }
 
 impl FileSlot {
-    /// Create a new path slot.
+    /// Create a new file slot.
     fn new(id: FileId) -> Self {
         Self { id, file: SlotCell::new(), source: SlotCell::new() }
     }

crates/typst-ide/Cargo.toml

@@ -21,5 +21,10 @@ log = { workspace = true }
 serde = { workspace = true }
 unscanny = { workspace = true }
 
+[dev-dependencies]
+typst-assets = { workspace = true }
+typst-dev-assets = { workspace = true }
+once_cell = { workspace = true }
+
 [lints]
 workspace = true

crates/typst-ide/src/complete.rs

@@ -1403,3 +1403,34 @@ impl<'a> CompletionContext<'a> {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use typst::eval::Tracer;
+
+    use super::autocomplete;
+    use crate::tests::TestWorld;
+
+    #[track_caller]
+    fn test(text: &str, cursor: usize, contains: &[&str], excludes: &[&str]) {
+        let world = TestWorld::new(text);
+        let doc = typst::compile(&world, &mut Tracer::new()).ok();
+        let (_, completions) =
+            autocomplete(&world, doc.as_ref(), &world.main, cursor, true)
+                .unwrap_or_default();
+
+        let labels: Vec<_> = completions.iter().map(|c| c.label.as_str()).collect();
+        for item in contains {
+            assert!(labels.contains(item), "{item:?} was not contained in {labels:?}");
+        }
+        for item in excludes {
+            assert!(!labels.contains(item), "{item:?} was not excluded in {labels:?}");
+        }
+    }
+
+    #[test]
+    fn test_autocomplete() {
+        test("#i", 2, &["int", "if conditional"], &["foo"]);
+        test("#().", 4, &["insert", "remove", "len", "all"], &["foo"]);
+    }
+}

crates/typst-ide/src/lib.rs

@@ -90,3 +90,88 @@ fn summarize_font_family<'a>(variants: impl Iterator<Item = &'a FontInfo>) -> Ec
 
     detail
 }
+
+#[cfg(test)]
+mod tests {
+    use comemo::Prehashed;
+    use once_cell::sync::Lazy;
+    use typst::diag::{FileError, FileResult};
+    use typst::foundations::{Bytes, Datetime};
+    use typst::syntax::{FileId, Source};
+    use typst::text::{Font, FontBook};
+    use typst::{Library, World};
+
+    /// A world for IDE testing.
+    pub struct TestWorld {
+        pub main: Source,
+        base: &'static TestBase,
+    }
+
+    impl TestWorld {
+        /// Create a new world for a single test.
+        ///
+        /// This is cheap because the shared base for all test runs is lazily
+        /// initialized just once.
+        pub fn new(text: &str) -> Self {
+            static BASE: Lazy<TestBase> = Lazy::new(TestBase::default);
+            let main = Source::detached(text);
+            Self { main, base: &*BASE }
+        }
+    }
+
+    impl World for TestWorld {
+        fn library(&self) -> &Prehashed<Library> {
+            &self.base.library
+        }
+
+        fn book(&self) -> &Prehashed<FontBook> {
+            &self.base.book
+        }
+
+        fn main(&self) -> Source {
+            self.main.clone()
+        }
+
+        fn source(&self, id: FileId) -> FileResult<Source> {
+            if id == self.main.id() {
+                Ok(self.main.clone())
+            } else {
+                Err(FileError::NotFound(id.vpath().as_rootless_path().into()))
+            }
+        }
+
+        fn file(&self, id: FileId) -> FileResult<Bytes> {
+            Err(FileError::NotFound(id.vpath().as_rootless_path().into()))
+        }
+
+        fn font(&self, index: usize) -> Option<Font> {
+            Some(self.base.fonts[index].clone())
+        }
+
+        fn today(&self, _: Option<i64>) -> Option<Datetime> {
+            None
+        }
+    }
+
+    /// Shared foundation of all test worlds.
+    struct TestBase {
+        library: Prehashed<Library>,
+        book: Prehashed<FontBook>,
+        fonts: Vec<Font>,
+    }
+
+    impl Default for TestBase {
+        fn default() -> Self {
+            let fonts: Vec<_> = typst_assets::fonts()
+                .chain(typst_dev_assets::fonts())
+                .flat_map(|data| Font::iter(Bytes::from_static(data)))
+                .collect();
+
+            Self {
+                library: Prehashed::new(Library::default()),
+                book: Prehashed::new(FontBook::from_fonts(&fonts)),
+                fonts,
+            }
+        }
+    }
+}

crates/typst-pdf/src/page.rs

@@ -377,7 +377,7 @@ pub struct EncodedPage {
 }
 
 /// Represents a resource being used in a PDF page by its name.
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd)]
 pub struct PageResource {
     kind: ResourceKind,
     name: EcoString,
@@ -390,7 +390,7 @@ impl PageResource {
 }
 
 /// A kind of resource being used in a PDF page.
-#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd)]
 pub enum ResourceKind {
     XObject,
     Font,

crates/typst-pdf/src/pattern.rs

@@ -118,13 +118,15 @@ fn register_pattern(
     // Render the body.
     let (_, content) = construct_page(ctx.parent, pattern.frame());
 
-    let pdf_pattern = PdfPattern {
+    let mut pdf_pattern = PdfPattern {
         transform,
         pattern: pattern.clone(),
         content: content.content.wait().clone(),
         resources: content.resources.into_iter().collect(),
     };
 
+    pdf_pattern.resources.sort();
+
     ctx.parent.pattern_map.insert(pdf_pattern)
 }
 

crates/typst-render/src/lib.rs

@@ -42,13 +42,13 @@ pub fn render(frame: &Frame, pixel_per_pt: f32, fill: Color) -> sk::Pixmap {
 
 /// Export a document with potentially multiple pages into a single raster image.
 ///
-/// The padding will be added around and between the individual frames.
+/// The gap will be added between the individual frames.
 pub fn render_merged(
     document: &Document,
     pixel_per_pt: f32,
     frame_fill: Color,
-    padding: Abs,
-    padding_fill: Color,
+    gap: Abs,
+    gap_fill: Color,
 ) -> sk::Pixmap {
     let pixmaps: Vec<_> = document
         .pages
@@ -56,19 +56,18 @@ pub fn render_merged(
         .map(|page| render(&page.frame, pixel_per_pt, frame_fill))
         .collect();
 
-    let padding = (pixel_per_pt * padding.to_f32()).round() as u32;
-    let pxw =
-        2 * padding + pixmaps.iter().map(sk::Pixmap::width).max().unwrap_or_default();
-    let pxh =
-        padding + pixmaps.iter().map(|pixmap| pixmap.height() + padding).sum::<u32>();
+    let gap = (pixel_per_pt * gap.to_f32()).round() as u32;
+    let pxw = pixmaps.iter().map(sk::Pixmap::width).max().unwrap_or_default();
+    let pxh = pixmaps.iter().map(|pixmap| pixmap.height()).sum::<u32>()
+        + gap * pixmaps.len().saturating_sub(1) as u32;
 
     let mut canvas = sk::Pixmap::new(pxw, pxh).unwrap();
-    canvas.fill(to_sk_color(padding_fill));
+    canvas.fill(to_sk_color(gap_fill));
 
-    let [x, mut y] = [padding; 2];
+    let mut y = 0;
     for pixmap in pixmaps {
         canvas.draw_pixmap(
-            x as i32,
+            0,
             y as i32,
             pixmap.as_ref(),
             &sk::PixmapPaint::default(),
@@ -76,7 +75,7 @@ pub fn render_merged(
             None,
         );
 
-        y += pixmap.height() + padding;
+        y += pixmap.height() + gap;
     }
 
     canvas

crates/typst/src/model/footnote.rs

@@ -167,11 +167,6 @@ cast! {
 /// This function is not intended to be called directly. Instead, it is used
 /// in set and show rules to customize footnote listings.
 ///
-/// _Note:_ Set and show rules for `footnote.entry` must be defined at the
-/// beginning of the document in order to work correctly.
-/// See [here](https://github.com/typst/typst/issues/1348#issuecomment-1566316463)
-/// for more information.
-///
 /// ```example
 /// #show footnote.entry: set text(red)
 ///
@@ -179,6 +174,12 @@ cast! {
 /// #footnote[It's down here]
 /// has red text!
 /// ```
+///
+/// _Note:_ Set and show rules for `footnote.entry` must be defined at the
+/// beginning of the document in order to work correctly. See [here][issue] for
+/// more information.
+///
+/// [issue]: https://github.com/typst/typst/issues/1467#issuecomment-1588799440
 #[elem(name = "entry", title = "Footnote Entry", Show, ShowSet)]
 pub struct FootnoteEntry {
     /// The footnote for this entry. It's location can be used to determine

tests/Cargo.toml

@@ -6,29 +6,29 @@ authors = { workspace = true }
 edition = { workspace = true }
 publish = false
 
-[dev-dependencies]
+[[test]]
+name = "tests"
+path = "src/tests.rs"
+harness = false
+
+[dependencies]
 typst = { workspace = true }
 typst-assets = { workspace = true, features = ["fonts"] }
 typst-dev-assets = { workspace = true }
 typst-pdf = { workspace = true }
 typst-render = { workspace = true }
 typst-svg = { workspace = true }
-typst-ide = { workspace = true }
 clap = { workspace = true }
 comemo = { workspace = true }
 ecow = { workspace = true }
 once_cell = { workspace = true }
 oxipng = { workspace = true }
+parking_lot = { workspace = true }
 rayon = { workspace = true }
 tiny-skia = { workspace = true }
 ttf-parser = { workspace = true }
 unscanny = { workspace = true }
 walkdir = { workspace = true }
 
-[[test]]
-name = "tests"
-path = "src/tests.rs"
-harness = false
-
 [lints]
 workspace = true

tests/README.md

@@ -3,13 +3,10 @@
 ## Directory structure
 Top level directory structure:
 - `src`: Testing code.
-- `typ`: Input files. The tests in `compiler` specifically test the compiler
-         while the others test the standard library (but also the compiler
-         indirectly).
+- `suite`: Input files. Mostly organize in parallel to the code.
 - `ref`: Reference images which the output is compared with to determine whether
          a test passed or failed.
-- `png`: PNG files produced by tests.
-- `pdf`: PDF files produced by tests.
+- `store`: Store for PNG, PDF, and SVG output files produced by the tests.
 
 ## Running the tests
 Running all tests (including unit tests):
@@ -37,11 +34,6 @@ Running a test with the exact filename `page.typ`.
 testit --exact page.typ
 ```
 
-Debug-printing the layout trees for all executed tests.
-```bash
-testit --debug empty.typ
-```
-
 To make the integration tests go faster they don't generate PDFs by default.
 Pass the `--pdf` flag to generate those. Mind that PDFs are not tested
 automatically at the moment, so you should always check the output manually when
@@ -50,18 +42,48 @@ making changes.
 testit --pdf
 ```
 
-## Update expected images
+## Writing tests
+The syntax for an individual test is `--- {name} ---` followed by some Typst
+code that should be tested. The name must be globally unique in the test suite,
+so that tests can be easily migrated across files.
+
+There are, broadly speaking, three kinds of tests:
+
+- Tests that just ensure that the code runs successfully: Those typically make
+  use of `test` or `assert.eq` (both are very similar, `test` is just shorter)
+  to ensure certain properties hold when executing the Typst code.
+
+- Tests that ensure the code fails with a particular error: Those have inline
+  annotations like `// Error: 2-7 thing was wrong`. An annotation can be
+  either an "Error", a "Warning", or a "Hint". The range designates where
+  in the next non-comment line the error is and after it follows the message.
+  If you the error is in a line further below, you can also write ranges like
+  `3:2-3:7` to indicate the 2-7 column in the 3rd non-comment line.
+
+- Tests that ensure certain visual output is produced: Those render the result
+  of the test with the `typst-render` crate and compare against a reference
+  image stored in the repository. The test runner automatically detects whether
+  a test has visual output and requires a reference image in this case.
+
+  To prevent bloat, it is important that the test images are kept as small as
+  possible. To that effect, the test runner enforces a maximum size of 20 KiB.
+  If truly necessary, this limit can however be lifted by adding `// LARGE` as
+  the first line of a test.
+
+If you have the choice between writing a test using assertions or using
+reference images, prefer assertions. This makes the test easier to understand
+in isolation and prevents bloat due to images.
+
+## Updating reference images
 If you created a new test or fixed a bug in an existing test, you need to update
-the reference image used for comparison. For this, you can use the
-`UPDATE_EXPECT` environment variable or the `--update` flag:
+the reference image used for comparison. For this, you can use the `--update`
+flag:
 ```bash
 testit mytest --update
 ```
 
 If you use the VS Code test helper extension (see the `tools` folder), you can
-alternatively use the checkmark button to update the reference image. In that
-case you should also install `oxipng` on your system so that the test helper
-can optimize the reference images.
+alternatively use the save button to update the reference image.
 
 ## Making an alias
 If you want to have a quicker way to run the tests, consider adding a shortcut