Add UPDATE_EXPECT envvar to update tests (#748)

tests/Cargo.toml

@@ -12,6 +12,7 @@ comemo = "0.2.2"
 elsa = "1.7"
 iai = { git = "https://github.com/reknih/iai" }
 once_cell = "1"
+oxipng = "8.0.0"
 tiny-skia = "0.6.2"
 ttf-parser = "0.17"
 unscanny = "0.1"

tests/README.md

@@ -50,18 +50,19 @@ making changes.
 testit --pdf
 ```
 
-## Creating new tests
-To keep things small, please optimize reference images before committing them.
-When you use the approve button from the Test Helper (see the `tools` folder)
-this happens automatically if you have `oxipng` installed.
+## Update expected images
+If you created a new test or fixed a bug in an existing test, you need to update
+the reference image used for comparison. For this, you can use the
+`UPDATE_EXPECT` environment varariable or the `--update` flag:
 ```bash
-# One image
-oxipng -o max path/to/image.png
-
-# All images
-oxipng -r -o max tests/ref
+testit mytest --update
 ```
 
+If you use the VS Code test helper extension (see the `tools` folder), you can
+alternatively use the checkmark button to update the reference image. In that
+case you should also install `oxipng` on your system so that the test helper
+can optimize the reference images.
+
 ## Making an alias
 If you want to have a quicker way to run the tests, consider adding a shortcut
 to your shell profile so that you can simply write something like:

tests/src/tests.rs

@@ -11,6 +11,7 @@ use std::path::{Path, PathBuf};
 use comemo::{Prehashed, Track};
 use elsa::FrozenVec;
 use once_cell::unsync::OnceCell;
+use oxipng::{InFile, Options, OutFile};
 use tiny_skia as sk;
 use typst::diag::{bail, FileError, FileResult};
 use typst::doc::{Document, Frame, FrameItem, Meta};
@@ -77,14 +78,27 @@ fn main() {
         let pdf_path =
             args.pdf.then(|| Path::new(PDF_DIR).join(path).with_extension("pdf"));
 
-        ok += test(&mut world, &src_path, &png_path, &ref_path, pdf_path.as_deref())
-            as usize;
+        ok += test(
+            &mut world,
+            &src_path,
+            &png_path,
+            &ref_path,
+            pdf_path.as_deref(),
+            args.update,
+        ) as usize;
     }
 
     if len > 1 {
         println!("{ok} / {len} tests passed.");
     }
 
+    if ok != len {
+        println!(
+            "Set the UPDATE_EXPECT environment variable or pass the \
+             --update flag to update the reference image(s)."
+        );
+    }
+
     if ok < len {
         std::process::exit(1);
     }
@@ -95,6 +109,7 @@ struct Args {
     filter: Vec<String>,
     exact: bool,
     pdf: bool,
+    update: bool,
     print: PrintConfig,
 }
 
@@ -111,6 +126,7 @@ impl Args {
         let mut filter = Vec::new();
         let mut exact = false;
         let mut pdf = false;
+        let mut update = env::var_os("UPDATE_EXPECT").is_some();
         let mut print = PrintConfig::default();
 
         for arg in args {
@@ -121,6 +137,8 @@ impl Args {
                 "--exact" => exact = true,
                 // Generate PDFs.
                 "--pdf" => pdf = true,
+                // Update the reference images.
+                "--update" => update = true,
                 // Debug print the syntax trees.
                 "--syntax" => print.syntax = true,
                 // Debug print the model.
@@ -132,7 +150,7 @@ impl Args {
             }
         }
 
-        Self { filter, exact, pdf, print }
+        Self { filter, exact, pdf, update, print }
     }
 
     fn matches(&self, path: &Path) -> bool {
@@ -339,6 +357,7 @@ fn test(
     png_path: &Path,
     ref_path: &Path,
     pdf_path: Option<&Path>,
+    update: bool,
 ) -> bool {
     let name = src_path.strip_prefix(TYP_DIR).unwrap_or(src_path);
     println!("Testing {}", name.display());
@@ -346,6 +365,7 @@ fn test(
     let text = fs::read_to_string(src_path).unwrap();
 
     let mut ok = true;
+    let mut updated = false;
     let mut frames = vec![];
     let mut line = 0;
     let mut compare_ref = true;
@@ -404,16 +424,26 @@ fn test(
                     .zip(ref_pixmap.data())
                     .any(|(&a, &b)| a.abs_diff(b) > 2)
             {
-                println!("  Does not match reference image. ❌");
-                ok = false;
+                if update {
+                    update_image(png_path, ref_path);
+                    updated = true;
+                } else {
+                    println!("  Does not match reference image. ❌");
+                    ok = false;
+                }
             }
         } else if !document.pages.is_empty() {
-            println!("  Failed to open reference image. ❌");
-            ok = false;
+            if update {
+                update_image(png_path, ref_path);
+                updated = true;
+            } else {
+                println!("  Failed to open reference image. ❌");
+                ok = false;
+            }
         }
     }
 
-    if ok {
+    if ok && !updated {
         if world.print == PrintConfig::default() {
             print!("\x1b[1A");
         }
@@ -423,6 +453,16 @@ fn test(
     ok
 }
 
+fn update_image(png_path: &Path, ref_path: &Path) {
+    println!("  Updated reference image. ✔");
+    oxipng::optimize(
+        &InFile::Path(png_path.to_owned()),
+        &OutFile::Path(Some(ref_path.to_owned())),
+        &Options::max_compression(),
+    )
+    .unwrap();
+}
+
 fn test_part(
     world: &mut TestWorld,
     src_path: &Path,

tools/test-helper/README.md

@@ -7,5 +7,8 @@ the `tests` folder.
 - Open: Opens the output and reference images of a test to the side.
 - Refresh: Refreshes the preview.
 - Rerun: Re-runs the test.
-- Approve: Copies the output into the reference folder and optimizes
+- Update: Copies the output into the reference folder and optimizes
   it with `oxipng`.
+
+For the test helper to work correctly, you also need to install `oxipng`, for
+example with `cargo install oxipng`.

tools/test-helper/extension.js

@@ -51,7 +51,7 @@ function activate(context) {
         )
     })
 
-    const approveCmd = vscode.commands.registerCommand("ShortcutMenuBar.testApprove", () => {
+    const updateCmd = vscode.commands.registerCommand("ShortcutMenuBar.testUpdate", () => {
         const uri = vscode.window.activeTextEditor.document.uri
         const { pngPath, refPath } = getPaths(uri)
 
@@ -66,7 +66,7 @@ function activate(context) {
     context.subscriptions.push(openCmd)
     context.subscriptions.push(refreshCmd)
     context.subscriptions.push(rerunCmd)
-    context.subscriptions.push(approveCmd)
+    context.subscriptions.push(updateCmd)
 }
 
 function getPaths(uri) {

tools/test-helper/package.json

@@ -1,7 +1,7 @@
 {
   "name": "typst-test-helper",
   "displayName": "Typst Test Helper",
-  "description": "Helps to run, compare and approve Typst tests.",
+  "description": "Helps to run, compare and update Typst tests.",
   "version": "0.0.1",
   "engines": {
     "vscode": "^1.53.0"
@@ -13,7 +13,7 @@
     "onCommand:ShortcutMenuBar.testOpen",
     "onCommand:ShortcutMenuBar.testRefresh",
     "onCommand:ShortcutMenuBar.testRerun",
-    "onCommand:ShortcutMenuBar.testApprove"
+    "onCommand:ShortcutMenuBar.testUpdate"
   ],
   "main": "./extension.js",
   "contributes": {
@@ -46,12 +46,12 @@
         }
       },
       {
-        "command": "ShortcutMenuBar.testApprove",
-        "title": "Approve output",
+        "command": "ShortcutMenuBar.testUpdate",
+        "title": "Update reference image",
         "category": "ShortcutMenuBar",
         "icon": {
-          "light": "images/approve-light.svg",
-          "dark": "images/approve-dark.svg"
+          "light": "images/update-light.svg",
+          "dark": "images/update-dark.svg"
         }
       }
     ],
@@ -74,7 +74,7 @@
         },
         {
           "when": "resourceExtname == .typ && resourcePath =~ /.*tests.*/",
-          "command": "ShortcutMenuBar.testApprove",
+          "command": "ShortcutMenuBar.testUpdate",
           "group": "navigation@3"
         }
       ]