Enhance the test runner: regex, --list, --path (#3945)

Cargo.lock

@@ -2780,6 +2780,7 @@ dependencies = [
  "oxipng",
  "parking_lot",
  "rayon",
+ "regex",
  "tiny-skia",
  "ttf-parser",
  "typst",

tests/Cargo.toml

@@ -25,6 +25,7 @@ once_cell = { workspace = true }
 oxipng = { workspace = true }
 parking_lot = { workspace = true }
 rayon = { workspace = true }
+regex = { workspace = true }
 tiny-skia = { workspace = true }
 ttf-parser = { workspace = true }
 unscanny = { workspace = true }

tests/README.md

@@ -3,7 +3,9 @@
 ## Directory structure
 Top level directory structure:
 - `src`: Testing code.
-- `suite`: Input files. Mostly organize in parallel to the code.
+- `suite`: Input files. Mostly organized in parallel to the code. Each file can
+           contain multiple tests, each of which is a section of Typst code
+           following `--- {name} ---`.
 - `ref`: Reference images which the output is compared with to determine whether
          a test passed or failed.
 - `store`: Store for PNG, PDF, and SVG output files produced by the tests.
@@ -19,17 +21,27 @@ Running just the integration tests (the tests in this directory):
 cargo test --workspace --test tests
 ```
 
-You may want to [make yourself an alias](#making-an-alias) like:
+You may want to [make yourself an alias](#making-an-alias) `testit` so that you can
+write shorter commands. In the examples below, we will use this alias.
+
+Running all tests with the given name pattern. You can use
+[regular expression](https://docs.rs/regex/latest/regex/)s.
 ```bash
-testit
+testit math            # The name has "math" anywhere
+testit math page       # The name has "math" or "page" anywhere
+testit "^math" "^page" # The name begins with "math" or "page"
+testit "^(math|page)"  # Same as above.
 ```
 
-Running all tests whose names contain the string `page` or `stack`. Note each
-`.typ` file in this directory can contain multiple tests, each of which is a
-section of Typst code following `--- {name} ---`.
+Running all tests discovered under given paths:
 ```bash
-# Add --verbose to list which tests were run.
-testit page stack
+testit -p tests/suite/math/attach.typ
+testit -p tests/suite/model -p tests/suite/text
+```
+
+Running tests that begin with `issue` under a given path:
+```bash
+testit "^issue" -p tests/suite/model
 ```
 
 Running a test with the exact test name `math-attach-mixed`.
@@ -37,6 +49,11 @@ Running a test with the exact test name `math-attach-mixed`.
 testit --exact math-attach-mixed
 ```
 
+You may find more options in the help message:
+```bash
+testit --help
+```
+
 To make the integration tests go faster they don't generate PDFs by default.
 Pass the `--pdf` flag to generate those. Mind that PDFs are not tested
 automatically at the moment, so you should always check the output manually when
@@ -56,12 +73,12 @@ There are, broadly speaking, three kinds of tests:
   use of `test` or `assert.eq` (both are very similar, `test` is just shorter)
   to ensure certain properties hold when executing the Typst code.
 
-- Tests that ensure the code fails with a particular error: Those have inline
-  annotations like `// Error: 2-7 thing was wrong`. An annotation can be
-  either an "Error", a "Warning", or a "Hint". The range designates where
-  in the next non-comment line the error is and after it follows the message.
-  If you the error is in a line further below, you can also write ranges like
-  `3:2-3:7` to indicate the 2-7 column in the 3rd non-comment line.
+- Tests that ensure the code emits particular diagnostic messages: Those have
+  inline annotations like `// Error: 2-7 thing was wrong`. An annotation can
+  start with either "Error", "Warning", or "Hint". The range designates the
+  code span the diagnostic message refers to in the first non-comment line
+  below. If the code span is in a line further below, you can write ranges
+  like `3:2-3:7` to indicate the 2-7 column in the 3rd non-comment line.
 
 - Tests that ensure certain visual output is produced: Those render the result
   of the test with the `typst-render` crate and compare against a reference
@@ -82,7 +99,7 @@ If you created a new test or fixed a bug in an existing test, you need to update
 the reference image used for comparison. For this, you can use the `--update`
 flag:
 ```bash
-testit mytest --update
+testit --exact my-test-name --update
 ```
 
 If you use the VS Code test helper extension (see the `tools` folder), you can
@@ -92,7 +109,7 @@ alternatively use the save button to update the reference image.
 If you want to have a quicker way to run the tests, consider adding a shortcut
 to your shell profile so that you can simply write something like:
 ```bash
-testit empty.typ
+testit --exact my-test-name
 ```
 
 ### Bash

tests/src/args.rs

@@ -1,19 +1,30 @@
+use std::path::PathBuf;
+
 use clap::{Parser, Subcommand};
+use regex::Regex;
 
 /// Typst's test runner.
 #[derive(Debug, Clone, Parser)]
+#[command(bin_name = "cargo test --workspace --test tests --")]
 #[clap(name = "typst-test", author)]
 pub struct CliArguments {
     /// The command to run.
     #[command(subcommand)]
     pub command: Option<Command>,
-    /// All the tests that contain the filter string will be run.
-    pub filter: Vec<String>,
-    /// Runs only the tests with the exact specified `filter` names.
+    /// All the tests whose names match the test name pattern will be run.
+    #[arg(value_parser = Regex::new)]
+    pub pattern: Vec<Regex>,
+    /// Restricts test selection within the given path.
+    #[arg(short, long, value_parser = |s: &str| PathBuf::from(s).canonicalize())]
+    pub path: Vec<PathBuf>,
+    /// Only selects the test that matches with the test name verbatim.
     #[arg(short, long)]
     pub exact: bool,
-    /// Whether to update the reference images of non-passing tests.
-    #[arg(short, long)]
+    /// Lists what tests will be run, without actually running them.
+    #[arg(long, group = "action")]
+    pub list: bool,
+    /// Updates the reference images of non-passing tests.
+    #[arg(short, long, group = "action")]
     pub update: bool,
     /// The scaling factor to render the output image with.
     ///
@@ -26,7 +37,7 @@ pub struct CliArguments {
     /// Exports SVG outputs into the artifact store.
     #[arg(long)]
     pub svg: bool,
-    /// Whether to display the syntax tree.
+    /// Displays the syntax tree.
     #[arg(long)]
     pub syntax: bool,
     /// Prevents the terminal from being cleared of test names.

tests/src/collect.rs

@@ -257,7 +257,7 @@ impl<'a> Parser<'a> {
                 self.collector.large.insert(name.clone());
             }
 
-            if !filtered(&name) {
+            if !selected(&name, self.path.canonicalize().unwrap()) {
                 self.collector.skipped += 1;
                 continue;
             }
@@ -383,14 +383,23 @@ impl<'a> Parser<'a> {
     }
 }
 
-/// Whether a test is within the filtered set.
-fn filtered(name: &str) -> bool {
+/// Whether a test is within the selected set to run.
+fn selected(name: &str, abs: PathBuf) -> bool {
+    let paths = &crate::ARGS.path;
+    if !paths.is_empty() && !paths.iter().any(|path| abs.starts_with(path)) {
+        return false;
+    }
+
     let exact = crate::ARGS.exact;
-    let filter = &crate::ARGS.filter;
-    filter.is_empty()
-        || filter
-            .iter()
-            .any(|v| if exact { name == v } else { name.contains(v) })
+    let patterns = &crate::ARGS.pattern;
+    patterns.is_empty()
+        || patterns.iter().any(|pattern: &regex::Regex| {
+            if exact {
+                name == pattern.as_str()
+            } else {
+                pattern.is_match(name)
+            }
+        })
 }
 
 /// An error in a test file.

tests/src/logger.rs

@@ -6,7 +6,7 @@ use crate::run::TestResult;
 
 /// Receives status updates by individual test runs.
 pub struct Logger<'a> {
-    filtered: usize,
+    selected: usize,
     passed: usize,
     failed: usize,
     skipped: usize,
@@ -19,9 +19,9 @@ pub struct Logger<'a> {
 
 impl<'a> Logger<'a> {
     /// Create a new logger.
-    pub fn new(filtered: usize, skipped: usize) -> Self {
+    pub fn new(selected: usize, skipped: usize) -> Self {
         Self {
-            filtered,
+            selected,
             passed: 0,
             failed: 0,
             skipped,
@@ -86,10 +86,10 @@ impl<'a> Logger<'a> {
 
     /// Prints a summary and returns whether the test suite passed.
     pub fn finish(&self) -> bool {
-        let Self { filtered, passed, failed, skipped, .. } = *self;
+        let Self { selected, passed, failed, skipped, .. } = *self;
 
         eprintln!("{passed} passed, {failed} failed, {skipped} skipped");
-        assert_eq!(filtered, passed + failed, "not all tests were executed succesfully");
+        assert_eq!(selected, passed + failed, "not all tests were executed succesfully");
 
         if self.mismatched_image {
             eprintln!("  pass the --update flag to update the reference images");
@@ -121,7 +121,7 @@ impl<'a> Logger<'a> {
 
         // Print the status line.
         let done = self.failed + self.passed;
-        if done < self.filtered {
+        if done < self.selected {
             if self.last_change.elapsed() > Duration::from_secs(2) {
                 for test in &self.active {
                     writeln!(out, "⏰ {test} is taking a long time ...")?;
@@ -131,7 +131,7 @@ impl<'a> Logger<'a> {
                 }
             }
             if self.terminal {
-                writeln!(out, "💨 {done} / {}", self.filtered)?;
+                writeln!(out, "💨 {done} / {}", self.selected)?;
                 self.temp_lines += 1;
             }
         }

tests/src/tests.rs

@@ -72,14 +72,20 @@ fn test() {
         }
     };
 
-    let filtered = tests.len();
-    if filtered == 0 {
+    let selected = tests.len();
+    if ARGS.list {
+        for test in tests.iter() {
+            println!("{test}");
+        }
+        eprintln!("{selected} selected, {skipped} skipped");
+        return;
+    } else if selected == 0 {
         eprintln!("no test selected");
         return;
     }
 
     // Run the tests.
-    let logger = Mutex::new(Logger::new(filtered, skipped));
+    let logger = Mutex::new(Logger::new(selected, skipped));
     std::thread::scope(|scope| {
         let logger = &logger;
         let (sender, receiver) = std::sync::mpsc::channel();