feat: [WIP] update test runner

tests/src/collect.rs

@@ -6,10 +6,13 @@ use std::str::FromStr;
 use std::sync::LazyLock;
 
 use ecow::{eco_format, EcoString};
+use typst::loading::LineCol;
 use typst_syntax::package::PackageVersion;
 use typst_syntax::{is_id_continue, is_ident, is_newline, FileId, Source, VirtualPath};
 use unscanny::Scanner;
 
+use crate::world::{read, system_path};
+
 /// Collects all tests from all files.
 ///
 /// Returns:
@@ -79,6 +82,8 @@ impl Display for FileSize {
 pub struct Note {
     pub pos: FilePos,
     pub kind: NoteKind,
+    /// The file [`Self::range`] belongs to.
+    pub file: FileId,
     pub range: Option<Range<usize>>,
     pub message: String,
 }
@@ -341,9 +346,31 @@ impl<'a> Parser<'a> {
         let kind: NoteKind = head.parse().ok()?;
         self.s.eat_if(' ');
 
+        let mut file = None;
+        if self.s.eat_if('"') {
+            let path = self.s.eat_until(|c| is_newline(c) || c == '"');
+            if !self.s.eat_if('"') {
+                self.error("expected closing quote after file path");
+                return None;
+            }
+
+            let vpath = VirtualPath::new(path);
+            file = Some(FileId::new(None, vpath));
+
+            self.s.eat_if(' ');
+        }
+
         let mut range = None;
-        if self.s.at('-') || self.s.at(char::is_numeric) {
+        if self.s.at('-') || self.s.at(char::is_numeric) || self.s.at('#') {
-            range = self.parse_range(source);
+            if let Some(file) = file {
+                range = self.parse_range_external(file);
+            } else if !self.s.at('#') {
+                range = self.parse_range(source);
+            } else {
+                self.error("raw byte positions are only allowed in external files");
+                return None;
+            }
+
             if range.is_none() {
                 self.error("range is malformed");
                 return None;
@@ -359,11 +386,79 @@ impl<'a> Parser<'a> {
         Some(Note {
             pos: FilePos::new(self.path, self.line),
             kind,
+            file: file.unwrap_or(source.id()),
             range,
             message,
         })
     }
 
+    /// Parse a range in an external file, optionally abbreviated as just a position
+    /// if the range is empty.
+    fn parse_range_external(&mut self, file: FileId) -> Option<Range<usize>> {
+        let path = match system_path(file) {
+            Ok(path) => path,
+            Err(err) => {
+                self.error(err.to_string());
+                return None;
+            }
+        };
+
+        let text = match read(&path) {
+            Ok(text) => text,
+            Err(err) => {
+                self.error(err.to_string());
+                return None;
+            }
+        };
+
+        // Allow parsing of byte positions for external files.
+        if self.s.peek() == Some('#') {
+            let start = self.parse_byte_position()?;
+            let end =
+                if self.s.eat_if('-') { self.parse_byte_position()? } else { start };
+
+            if start < 0 || end < 0 {
+                self.error("byte positions must be positive");
+                return None;
+            }
+
+            return Some((start as usize)..(end as usize));
+        }
+
+        let start = self.parse_line_col()?;
+        let range = if self.s.eat_if('-') {
+            let end = self.parse_line_col()?;
+            LineCol::byte_range(start..end, &text)
+        } else {
+            start.byte_pos(&text).map(|i| i..i)
+        };
+        if range.is_none() {
+            self.error("range is out of bounds");
+        }
+        range
+    }
+
+    /// Parses an absolute `line:column` position in an external file.
+    fn parse_line_col(&mut self) -> Option<LineCol> {
+        let line = self.parse_number()?;
+        if !self.s.eat_if(':') {
+            self.error("positions in external files always require both `<line>:<col>`");
+            return None;
+        }
+        let col = self.parse_number()?;
+        if line < 0 || col < 0 {
+            self.error("line and column numbers must be positive");
+            return None;
+        }
+
+        Some(LineCol::one_based(line as usize, col as usize))
+    }
+
+    /// Parses a number after a `#` character.
+    fn parse_byte_position(&mut self) -> Option<isize> {
+        self.s.eat_if("#").then(|| self.parse_number()).flatten()
+    }
+
     /// Parse a range, optionally abbreviated as just a position if the range
     /// is empty.
     fn parse_range(&mut self, source: &Source) -> Option<Range<usize>> {

tests/src/run.rs

@@ -7,13 +7,15 @@ use tiny_skia as sk;
 use typst::diag::{SourceDiagnostic, Warned};
 use typst::html::HtmlDocument;
 use typst::layout::{Abs, Frame, FrameItem, PagedDocument, Transform};
+use typst::loading::LineCol;
 use typst::visualize::Color;
-use typst::{Document, WorldExt};
+use typst::{Document, World, WorldExt};
 use typst_pdf::PdfOptions;
+use typst_syntax::FileId;
 
 use crate::collect::{Attr, FileSize, NoteKind, Test};
 use crate::logger::TestResult;
-use crate::world::TestWorld;
+use crate::world::{system_path, TestWorld};
 
 /// Runs a single test.
 ///
@@ -117,7 +119,7 @@ impl<'a> Runner<'a> {
             if seen {
                 continue;
             }
-            let note_range = self.format_range(&note.range);
+            let note_range = self.format_range(note.file, &note.range);
             if first {
                 log!(self, "not emitted");
                 first = false;
@@ -208,10 +210,6 @@ impl<'a> Runner<'a> {
     /// Compare a subset of notes with a given kind against diagnostics of
     /// that same kind.
     fn check_diagnostic(&mut self, kind: NoteKind, diag: &SourceDiagnostic) {
-        // Ignore diagnostics from other sources than the test file itself.
-        if diag.span.id().is_some_and(|id| id != self.test.source.id()) {
-            return;
-        }
         // TODO: remove this once HTML export is stable
         if diag.message == "html export is under active development and incomplete" {
             return;
@@ -219,11 +217,11 @@ impl<'a> Runner<'a> {
 
         let message = diag.message.replace("\\", "/");
         let range = self.world.range(diag.span);
-        self.validate_note(kind, range.clone(), &message);
+        self.validate_note(kind, diag.span.id(), range.clone(), &message);
 
         // Check hints.
         for hint in &diag.hints {
-            self.validate_note(NoteKind::Hint, range.clone(), hint);
+            self.validate_note(NoteKind::Hint, diag.span.id(), range.clone(), hint);
         }
     }
 
@@ -235,15 +233,18 @@ impl<'a> Runner<'a> {
     fn validate_note(
         &mut self,
         kind: NoteKind,
+        file: Option<FileId>,
         range: Option<Range<usize>>,
         message: &str,
     ) {
         // Try to find perfect match.
+        let file = file.unwrap_or(self.test.source.id());
         if let Some((i, _)) = self.test.notes.iter().enumerate().find(|&(i, note)| {
             !self.seen[i]
                 && note.kind == kind
                 && note.range == range
                 && note.message == message
+                && note.file == file
         }) {
             self.seen[i] = true;
             return;
@@ -257,7 +258,7 @@ impl<'a> Runner<'a> {
                 && (note.range == range || note.message == message)
         }) else {
             // Not even a close match, diagnostic is not annotated.
-            let diag_range = self.format_range(&range);
+            let diag_range = self.format_range(file, &range);
             log!(into: self.not_annotated, "  {kind}: {diag_range} {}", message);
             return;
         };
@@ -267,10 +268,10 @@ impl<'a> Runner<'a> {
 
         // Range is wrong.
         if range != note.range {
-            let note_range = self.format_range(&note.range);
+            let note_range = self.format_range(note.file, &note.range);
-            let note_text = self.text_for_range(&note.range);
+            let note_text = self.text_for_range(note.file, &note.range);
-            let diag_range = self.format_range(&range);
+            let diag_range = self.format_range(note.file, &range);
-            let diag_text = self.text_for_range(&range);
+            let diag_text = self.text_for_range(note.file, &range);
             log!(self, "mismatched range ({}):", note.pos);
             log!(self, "  message   | {}", note.message);
             log!(self, "  annotated | {note_range:<9} | {note_text}");
@@ -286,39 +287,63 @@ impl<'a> Runner<'a> {
     }
 
     /// Display the text for a range.
-    fn text_for_range(&self, range: &Option<Range<usize>>) -> String {
+    fn text_for_range(&self, file: FileId, range: &Option<Range<usize>>) -> String {
         let Some(range) = range else { return "No text".into() };
         if range.is_empty() {
             "(empty)".into()
+        } else if file == self.test.source.id() {
+            format!(
+                "`{}`",
+                self.test.source.text()[range.clone()]
+                    .replace('\n', "\\n")
+                    .replace('\r', "\\r")
+            )
         } else {
-            format!("`{}`", self.test.source.text()[range.clone()].replace('\n', "\\n"))
+            let bytes = self.world.file(file).unwrap();
+            let text = String::from_utf8_lossy(&bytes);
+            format!("`{}`", text[range.clone()].replace('\n', "\\n").replace('\r', "\\r"))
         }
     }
 
     /// Display a byte range as a line:column range.
-    fn format_range(&self, range: &Option<Range<usize>>) -> String {
+    fn format_range(&self, file: FileId, range: &Option<Range<usize>>) -> String {
         let Some(range) = range else { return "No range".into() };
+
+        let mut preamble = String::new();
+        if file != self.test.source.id() {
+            preamble = format!("\"{}\" ", system_path(file).unwrap().display());
+        }
+
         if range.start == range.end {
-            self.format_pos(range.start)
+            format!("{preamble}{}", self.format_pos(file, range.start))
         } else {
-            format!("{}-{}", self.format_pos(range.start,), self.format_pos(range.end,))
+            format!(
+                "{preamble}{}-{}",
+                self.format_pos(file, range.start),
+                self.format_pos(file, range.end)
+            )
         }
     }
 
     /// Display a position as a line:column pair.
-    fn format_pos(&self, pos: usize) -> String {
+    fn format_pos(&self, file: FileId, pos: usize) -> String {
-        if let (Some(line_idx), Some(column_idx)) =
+        let res = if file != self.test.source.id() {
-            (self.test.source.byte_to_line(pos), self.test.source.byte_to_column(pos))
+            let bytes = self.world.file(file).unwrap();
-        {
+            LineCol::from_byte_pos(pos, &bytes).map(|l| l.numbers())
-            let line = self.test.pos.line + line_idx;
-            let column = column_idx + 1;
-            if line == 1 {
-                format!("{column}")
-            } else {
-                format!("{line}:{column}")
-            }
         } else {
-            "oob".into()
+            let line = self.test.source.byte_to_line(pos).map(|l| l + 1);
+            let col = (self.test.source.byte_to_column(pos))
+                .map(|c| self.test.pos.line + c + 1);
+            Option::zip(line, col)
+        };
+        let Some((line, col)) = res else {
+            return "oob".into();
+        };
+
+        if line == 1 {
+            format!("{col}")
+        } else {
+            format!("{line}:{col}")
         }
     }
 }

tests/src/world.rs

@@ -149,7 +149,7 @@ impl FileSlot {
 }
 
 /// The file system path for a file ID.
-fn system_path(id: FileId) -> FileResult<PathBuf> {
+pub(crate) fn system_path(id: FileId) -> FileResult<PathBuf> {
     let root: PathBuf = match id.package() {
         Some(spec) => format!("tests/packages/{}-{}", spec.name, spec.version).into(),
         None => PathBuf::new(),
@@ -159,7 +159,7 @@ fn system_path(id: FileId) -> FileResult<PathBuf> {
 }
 
 /// Read a file.
-fn read(path: &Path) -> FileResult<Cow<'static, [u8]>> {
+pub(crate) fn read(path: &Path) -> FileResult<Cow<'static, [u8]>> {
     // Resolve asset.
     if let Ok(suffix) = path.strip_prefix("assets/") {
         return typst_dev_assets::get(&suffix.to_string_lossy())

tests/suite/loading/csv.typ

@@ -18,12 +18,12 @@
 #csv("nope.csv")
 
 --- csv-invalid ---
-// Error: 6-28 failed to parse CSV (found 3 instead of 2 fields in line 3)
+// Error: "/assets/data/bad.csv" 3:1 failed to parse CSV (found 3 instead of 2 fields in line 3)
 #csv("/assets/data/bad.csv")
 
 --- csv-invalid-row-type-dict ---
 // Test error numbering with dictionary rows.
-// Error: 6-28 failed to parse CSV (found 3 instead of 2 fields in line 3)
+// Error: "/assets/data/bad.csv" 3:1 failed to parse CSV (found 3 instead of 2 fields in line 3)
 #csv("/assets/data/bad.csv", row-type: dictionary)
 
 --- csv-invalid-delimiter ---

tests/suite/loading/json.typ

@@ -6,7 +6,7 @@
 #test(data.at(2).weight, 150)
 
 --- json-invalid ---
-// Error: 7-30 failed to parse JSON (expected value at line 3 column 14)
+// Error: "/assets/data/bad.json" 3:14 failed to parse JSON (expected value at line 3 column 14)
 #json("/assets/data/bad.json")
 
 --- json-decode-deprecated ---

tests/suite/loading/toml.typ

@@ -37,7 +37,7 @@
 ))
 
 --- toml-invalid ---
-// Error: 7-30 failed to parse TOML (expected `.`, `=` at line 1 column 16)
+// Error: "/assets/data/bad.toml" #15-#16 failed to parse TOML (expected `.`, `=`)
 #toml("/assets/data/bad.toml")
 
 --- toml-decode-deprecated ---

tests/suite/loading/xml.typ

@@ -24,7 +24,7 @@
 ),))
 
 --- xml-invalid ---
-// Error: 6-28 failed to parse XML (found closing tag 'data' instead of 'hello' in line 3)
+// Error: "/assets/data/bad.xml" 3:0 failed to parse XML (found closing tag 'data' instead of 'hello')
 #xml("/assets/data/bad.xml")
 
 --- xml-decode-deprecated ---

tests/suite/loading/yaml.typ

@@ -13,7 +13,7 @@
 #test(data.at("1"), "ok")
 
 --- yaml-invalid ---
-// Error: 7-30 failed to parse YAML (did not find expected ',' or ']' at line 2 column 1, while parsing a flow sequence at line 1 column 18)
+// Error: "/assets/data/bad.yaml" 2:1 failed to parse YAML (did not find expected ',' or ']' at line 2 column 1, while parsing a flow sequence at line 1 column 18)
 #yaml("/assets/data/bad.yaml")
 
 --- yaml-decode-deprecated ---

tests/suite/scripting/import.typ

@@ -334,6 +334,7 @@
 
 --- import-cyclic-in-other-file ---
 // Cyclic import in other file.
+// error: "./modules/cycle1.typ" 2:29-2:51 file is not valid utf-8
 #import "./modules/cycle1.typ": *
 
 This is never reached.

tests/suite/visualize/image.typ

@@ -167,7 +167,7 @@ A #box(image("/assets/images/tiger.jpg", height: 1cm, width: 80%)) B
 #image("/assets/plugins/hello.wasm")
 
 --- image-bad-svg ---
-// Error: 2-33 failed to parse SVG (found closing tag 'g' instead of 'style' in line 4)
+// Error: "/assets/images/bad.svg" 4:0 failed to parse SVG (found closing tag 'g' instead of 'style' at 4:3)
 #image("/assets/images/bad.svg")
 
 --- image-decode-svg ---