diff --git a/tests/src/args.rs b/tests/src/args.rs index 786733cce..e94986ced 100644 --- a/tests/src/args.rs +++ b/tests/src/args.rs @@ -43,7 +43,9 @@ pub struct CliArguments { /// Runs SVG export. #[arg(long)] pub svg: bool, - /// Displays the syntax tree. + /// Displays the syntax tree before running tests. + /// + /// Note: This is ignored if using '--syntax-compare'. #[arg(long)] pub syntax: bool, /// Displays only one line per test, hiding details about failures. @@ -55,6 +57,25 @@ pub struct CliArguments { /// How many threads to spawn when running the tests. #[arg(short = 'j', long)] pub num_threads: Option, + /// Changes testing behavior for debugging the parser: With no argument, + /// outputs the concrete syntax trees of tests as files in + /// 'tests/store/syntax/'. With a directory as argument, will treat it as a + /// reference of correct syntax tree files and will print which output + /// syntax trees differ (viewing the diffs is on you). + /// + /// This overrides the normal testing system. It parses, but does not run + /// the test suite. + /// + /// You can generate a correct reference directory by running on a known + /// good commit and copying the generated outputs to a new directory. + /// `_things` may be a good location as it is in the top-level gitignore. + /// + /// You can view diffs in VS Code with: `code --diff /.syntax + /// tests/store/syntax/.syntax` + #[arg(long)] + pub parser_compare: Option>, + // ^ I'm not using a subcommand here because then test patterns don't parse + // how you would expect and I'm too lazy to try to fix it. } impl CliArguments { diff --git a/tests/src/tests.rs b/tests/src/tests.rs index 940c9e3c4..eb2cfd796 100644 --- a/tests/src/tests.rs +++ b/tests/src/tests.rs @@ -7,7 +7,7 @@ mod logger; mod run; mod world; -use std::path::Path; +use std::path::{Path, PathBuf}; use std::sync::LazyLock; use std::time::Duration; @@ -16,7 +16,9 @@ use parking_lot::Mutex; use rayon::iter::{ParallelBridge, ParallelIterator}; use crate::args::{CliArguments, Command}; +use crate::collect::Test; use crate::logger::Logger; +use crate::run::TestResult; /// The parsed command line arguments. static ARGS: LazyLock = LazyLock::new(CliArguments::parse); @@ -27,6 +29,9 @@ const SUITE_PATH: &str = "tests/suite"; /// The directory where the full test results are stored. const STORE_PATH: &str = "tests/store"; +/// The directory where syntax trees are stored. +const SYNTAX_PATH: &str = "tests/store/syntax"; + /// The directory where the reference images are stored. const REF_PATH: &str = "tests/ref"; @@ -89,6 +94,16 @@ fn test() { return; } + let parser_dirs = ARGS.parser_compare.clone().map(create_syntax_store); + + let runner = |test: &Test| { + if let Some((live_path, ref_path)) = &parser_dirs { + run_parser_test(test, live_path, ref_path) + } else { + run::run(test) + } + }; + // Run the tests. let logger = Mutex::new(Logger::new(selected, skipped)); std::thread::scope(|scope| { @@ -112,7 +127,7 @@ fn test() { // to `typst::utils::Deferred` yielding. tests.iter().par_bridge().for_each(|test| { logger.lock().start(test); - let result = std::panic::catch_unwind(|| run::run(test)); + let result = std::panic::catch_unwind(|| runner(test)); logger.lock().end(test, result); }); @@ -142,3 +157,46 @@ fn undangle() { } } } + +fn create_syntax_store(ref_path: Option) -> (&'static Path, Option) { + if ref_path.as_ref().is_some_and(|p| !p.exists()) { + eprintln!("syntax reference path doesn't exist"); + std::process::exit(1); + } + + let live_path = Path::new(SYNTAX_PATH); + std::fs::remove_dir_all(live_path).ok(); + std::fs::create_dir_all(live_path).unwrap(); + (live_path, ref_path) +} + +fn run_parser_test( + test: &Test, + live_path: &Path, + ref_path: &Option, +) -> TestResult { + let mut result = TestResult { + errors: String::new(), + infos: String::new(), + mismatched_image: false, + }; + + let syntax_file = live_path.join(format!("{}.syntax", test.name)); + let tree = format!("{:#?}\n", test.source.root()); + std::fs::write(syntax_file, &tree).unwrap(); + + let Some(ref_path) = ref_path else { return result }; + let ref_file = ref_path.join(format!("{}.syntax", test.name)); + match std::fs::read_to_string(&ref_file) { + Ok(ref_tree) => { + if tree != ref_tree { + result.errors = "differs".to_string(); + } + } + Err(_) => { + result.errors = format!("missing reference: {}", ref_file.display()); + } + } + + result +}