Better error message for compile time string interning failure (#6439)

Co-authored-by: Laurenz <laurmaedje@gmail.com>

crates/typst-cli/src/query.rs

@@ -5,9 +5,9 @@ use typst::diag::{bail, HintedStrResult, StrResult, Warned};
 use typst::engine::Sink;
 use typst::foundations::{Content, IntoValue, LocatableSelector, Scope};
 use typst::layout::PagedDocument;
-use typst::syntax::Span;
+use typst::syntax::{Span, SyntaxMode};
 use typst::World;
-use typst_eval::{eval_string, EvalMode};
+use typst_eval::eval_string;
 
 use crate::args::{QueryCommand, SerializationFormat};
 use crate::compile::print_diagnostics;
@@ -63,7 +63,7 @@ fn retrieve(
         Sink::new().track_mut(),
         &command.selector,
         Span::detached(),
-        EvalMode::Code,
+        SyntaxMode::Code,
         Scope::default(),
     )
     .map_err(|errors| {

crates/typst-eval/src/lib.rs

@@ -18,7 +18,6 @@ pub use self::call::{eval_closure, CapturesVisitor};
 pub use self::flow::FlowEvent;
 pub use self::import::import;
 pub use self::vm::Vm;
-pub use typst_library::routines::EvalMode;
 
 use self::access::*;
 use self::binding::*;
@@ -32,7 +31,7 @@ use typst_library::introspection::Introspector;
 use typst_library::math::EquationElem;
 use typst_library::routines::Routines;
 use typst_library::World;
-use typst_syntax::{ast, parse, parse_code, parse_math, Source, Span};
+use typst_syntax::{ast, parse, parse_code, parse_math, Source, Span, SyntaxMode};
 
 /// Evaluate a source file and return the resulting module.
 #[comemo::memoize]
@@ -104,13 +103,13 @@ pub fn eval_string(
     sink: TrackedMut<Sink>,
     string: &str,
     span: Span,
-    mode: EvalMode,
+    mode: SyntaxMode,
     scope: Scope,
 ) -> SourceResult<Value> {
     let mut root = match mode {
-        EvalMode::Code => parse_code(string),
+        SyntaxMode::Code => parse_code(string),
-        EvalMode::Markup => parse(string),
+        SyntaxMode::Markup => parse(string),
-        EvalMode::Math => parse_math(string),
+        SyntaxMode::Math => parse_math(string),
     };
 
     root.synthesize(span);
@@ -141,11 +140,11 @@ pub fn eval_string(
 
     // Evaluate the code.
     let output = match mode {
-        EvalMode::Code => root.cast::<ast::Code>().unwrap().eval(&mut vm)?,
+        SyntaxMode::Code => root.cast::<ast::Code>().unwrap().eval(&mut vm)?,
-        EvalMode::Markup => {
+        SyntaxMode::Markup => {
             Value::Content(root.cast::<ast::Markup>().unwrap().eval(&mut vm)?)
         }
-        EvalMode::Math => Value::Content(
+        SyntaxMode::Math => Value::Content(
             EquationElem::new(root.cast::<ast::Math>().unwrap().eval(&mut vm)?)
                 .with_block(false)
                 .pack()

crates/typst-library/src/foundations/cast.rs

@@ -9,7 +9,7 @@ use std::ops::Add;
 
 use ecow::eco_format;
 use smallvec::SmallVec;
-use typst_syntax::{Span, Spanned};
+use typst_syntax::{Span, Spanned, SyntaxMode};
 use unicode_math_class::MathClass;
 
 use crate::diag::{At, HintedStrResult, HintedString, SourceResult, StrResult};
@@ -459,6 +459,21 @@ impl FromValue for Never {
     }
 }
 
+cast! {
+    SyntaxMode,
+    self => IntoValue::into_value(match self {
+        SyntaxMode::Markup => "markup",
+        SyntaxMode::Math => "math",
+        SyntaxMode::Code => "code",
+    }),
+    /// Evaluate as markup, as in a Typst file.
+    "markup" => SyntaxMode::Markup,
+    /// Evaluate as math, as in an equation.
+    "math" => SyntaxMode::Math,
+    /// Evaluate as code, as after a hash.
+    "code" => SyntaxMode::Code,
+}
+
 cast! {
     MathClass,
     self => IntoValue::into_value(match self {

crates/typst-library/src/foundations/mod.rs

@@ -69,6 +69,7 @@ pub use self::ty::*;
 pub use self::value::*;
 pub use self::version::*;
 pub use typst_macros::{scope, ty};
+use typst_syntax::SyntaxMode;
 
 #[rustfmt::skip]
 #[doc(hidden)]
@@ -83,7 +84,6 @@ use typst_syntax::Spanned;
 
 use crate::diag::{bail, SourceResult, StrResult};
 use crate::engine::Engine;
-use crate::routines::EvalMode;
 use crate::{Feature, Features};
 
 /// Hook up all `foundations` definitions.
@@ -273,8 +273,8 @@ pub fn eval(
     /// #eval("1_2^3", mode: "math")
     /// ```
     #[named]
-    #[default(EvalMode::Code)]
+    #[default(SyntaxMode::Code)]
-    mode: EvalMode,
+    mode: SyntaxMode,
     /// A scope of definitions that are made available.
     ///
     /// ```example

crates/typst-library/src/model/bibliography.rs

@@ -16,7 +16,7 @@ use hayagriva::{
 };
 use indexmap::IndexMap;
 use smallvec::{smallvec, SmallVec};
-use typst_syntax::{Span, Spanned};
+use typst_syntax::{Span, Spanned, SyntaxMode};
 use typst_utils::{Get, ManuallyHash, NonZeroExt, PicoStr};
 
 use crate::diag::{
@@ -39,7 +39,7 @@ use crate::model::{
     CitationForm, CiteGroup, Destination, FootnoteElem, HeadingElem, LinkElem, ParElem,
     Url,
 };
-use crate::routines::{EvalMode, Routines};
+use crate::routines::Routines;
 use crate::text::{
     FontStyle, Lang, LocalName, Region, Smallcaps, SubElem, SuperElem, TextElem,
     WeightDelta,
@@ -1024,7 +1024,7 @@ impl ElemRenderer<'_> {
             Sink::new().track_mut(),
             math,
             self.span,
-            EvalMode::Math,
+            SyntaxMode::Math,
             Scope::new(),
         )
         .map(Value::display)

crates/typst-library/src/pdf/embed.rs

@@ -59,7 +59,7 @@ pub struct EmbedElem {
     // We can't distinguish between the two at the moment.
     #[required]
     #[parse(
-        match args.find::<Bytes>()? {
+        match args.eat::<Bytes>()? {
             Some(data) => data,
             None => engine.world.file(id).at(span)?,
         }

crates/typst-library/src/routines.rs

@@ -4,7 +4,7 @@ use std::hash::{Hash, Hasher};
 use std::num::NonZeroUsize;
 
 use comemo::{Tracked, TrackedMut};
-use typst_syntax::Span;
+use typst_syntax::{Span, SyntaxMode};
 use typst_utils::LazyHash;
 
 use crate::diag::SourceResult;
@@ -58,7 +58,7 @@ routines! {
         sink: TrackedMut<Sink>,
         string: &str,
         span: Span,
-        mode: EvalMode,
+        mode: SyntaxMode,
         scope: Scope,
     ) -> SourceResult<Value>
 
@@ -312,17 +312,6 @@ routines! {
     ) -> SourceResult<Fragment>
 }
 
-/// In which mode to evaluate a string.
-#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash, Cast)]
-pub enum EvalMode {
-    /// Evaluate as code, as after a hash.
-    Code,
-    /// Evaluate as markup, like in a Typst file.
-    Markup,
-    /// Evaluate as math, as in an equation.
-    Math,
-}
-
 /// Defines what kind of realization we are performing.
 pub enum RealizationKind<'a> {
     /// This the root realization for layout. Requires a mutable reference

crates/typst-library/src/text/lang.rs

@@ -14,13 +14,14 @@ macro_rules! translation {
     };
 }
 
-const TRANSLATIONS: [(&str, &str); 40] = [
+const TRANSLATIONS: &[(&str, &str)] = &[
     translation!("ar"),
     translation!("bg"),
     translation!("ca"),
     translation!("cs"),
     translation!("da"),
     translation!("de"),
+    translation!("el"),
     translation!("en"),
     translation!("es"),
     translation!("et"),
@@ -28,7 +29,6 @@ const TRANSLATIONS: [(&str, &str); 40] = [
     translation!("fi"),
     translation!("fr"),
     translation!("gl"),
-    translation!("el"),
     translation!("he"),
     translation!("hu"),
     translation!("id"),
@@ -41,8 +41,8 @@ const TRANSLATIONS: [(&str, &str); 40] = [
     translation!("nl"),
     translation!("nn"),
     translation!("pl"),
-    translation!("pt-PT"),
     translation!("pt"),
+    translation!("pt-PT"),
     translation!("ro"),
     translation!("ru"),
     translation!("sl"),
@@ -53,8 +53,8 @@ const TRANSLATIONS: [(&str, &str); 40] = [
     translation!("tr"),
     translation!("uk"),
     translation!("vi"),
-    translation!("zh-TW"),
     translation!("zh"),
+    translation!("zh-TW"),
 ];
 
 /// An identifier for a natural language.
@@ -312,14 +312,74 @@ fn lang_str(lang: Lang, region: Option<Region>) -> EcoString {
 
 #[cfg(test)]
 mod tests {
+    use std::collections::HashSet;
+    use std::path::PathBuf;
+
     use typst_utils::option_eq;
 
     use super::*;
 
+    fn translation_files_iter() -> impl Iterator<Item = PathBuf> {
+        std::fs::read_dir("translations")
+            .unwrap()
+            .map(|e| e.unwrap().path())
+            .filter(|e| e.is_file() && e.extension().is_some_and(|e| e == "txt"))
+    }
+
     #[test]
     fn test_region_option_eq() {
         let region = Some(Region([b'U', b'S']));
         assert!(option_eq(region, "US"));
         assert!(!option_eq(region, "AB"));
     }
+
+    #[test]
+    fn test_all_translations_included() {
+        let defined_keys =
+            HashSet::<&str>::from_iter(TRANSLATIONS.iter().map(|(lang, _)| *lang));
+        let mut checked = 0;
+        for file in translation_files_iter() {
+            assert!(
+                defined_keys.contains(
+                    file.file_stem()
+                        .expect("translation file should have basename")
+                        .to_str()
+                        .expect("translation file name should be utf-8 encoded")
+                ),
+                "translation from {:?} should be registered in TRANSLATIONS in {}",
+                file.file_name().unwrap(),
+                file!(),
+            );
+            checked += 1;
+        }
+        assert_eq!(TRANSLATIONS.len(), checked);
+    }
+
+    #[test]
+    fn test_all_translation_files_formatted() {
+        for file in translation_files_iter() {
+            let content = std::fs::read_to_string(&file)
+                .expect("translation file should be in utf-8 encoding");
+            let filename = file.file_name().unwrap();
+            assert!(
+                content.ends_with('\n'),
+                "translation file {filename:?} should end with linebreak",
+            );
+            for line in content.lines() {
+                assert_eq!(
+                    line.trim(),
+                    line,
+                    "line {line:?} in {filename:?} should not have extra whitespaces"
+                );
+            }
+        }
+    }
+
+    #[test]
+    fn test_translations_sorted() {
+        assert!(
+            TRANSLATIONS.is_sorted_by_key(|(lang, _)| lang),
+            "TRANSLATIONS should be sorted"
+        );
+    }
 }

crates/typst-library/translations/ar.txt

@@ -5,4 +5,4 @@ bibliography = المراجع
 heading = الفصل
 outline = المحتويات
 raw = قائمة
-page = صفحة
+page = صفحة

crates/typst-library/translations/bg.txt

@@ -5,4 +5,4 @@ bibliography = Библиография
 heading = Раздел
 outline = Съдържание
 raw = Приложение
-page = стр.
+page = стр.

crates/typst-library/translations/ca.txt

@@ -5,4 +5,4 @@ bibliography = Bibliografia
 heading = Secció
 outline = Índex
 raw = Llistat
-page = pàgina
+page = pàgina

crates/typst-library/translations/cs.txt

@@ -5,4 +5,4 @@ bibliography = Bibliografie
 heading = Kapitola
 outline = Obsah
 raw = Výpis
-page = strana
+page = strana

crates/typst-library/translations/da.txt

@@ -5,4 +5,4 @@ bibliography = Bibliografi
 heading = Afsnit
 outline = Indhold
 raw = Liste
-page = side
+page = side

crates/typst-library/translations/de.txt

@@ -5,4 +5,4 @@ bibliography = Bibliographie
 heading = Abschnitt
 outline = Inhaltsverzeichnis
 raw = Listing
-page = Seite
+page = Seite

crates/typst-library/translations/el.txt

@@ -4,4 +4,4 @@ equation = Εξίσωση
 bibliography = Βιβλιογραφία
 heading = Κεφάλαιο
 outline = Περιεχόμενα
-raw = Παράθεση
+raw = Παράθεση

crates/typst-library/translations/en.txt

@@ -5,4 +5,4 @@ bibliography = Bibliography
 heading = Section
 outline = Contents
 raw = Listing
-page = page
+page = page

crates/typst-library/translations/es.txt

@@ -5,4 +5,4 @@ bibliography = Bibliografía
 heading = Sección
 outline = Índice
 raw = Listado
-page = página
+page = página

crates/typst-library/translations/et.txt

@@ -5,4 +5,4 @@ bibliography = Viited
 heading = Peatükk
 outline = Sisukord
 raw = List
-page = lk.
+page = lk.

crates/typst-library/translations/fi.txt

@@ -5,4 +5,4 @@ bibliography = Viitteet
 heading = Osio
 outline = Sisällys
 raw = Esimerkki
-page = sivu
+page = sivu

crates/typst-library/translations/fr.txt

@@ -5,4 +5,4 @@ bibliography = Bibliographie
 heading = Chapitre
 outline = Table des matières
 raw = Liste
-page = page
+page = page

crates/typst-library/translations/gl.txt

@@ -5,4 +5,4 @@ bibliography = Bibliografía
 heading = Sección
 outline = Índice
 raw = Listado
-page = páxina
+page = páxina

crates/typst-library/translations/he.txt

@@ -5,4 +5,4 @@ bibliography = רשימת מקורות
 heading = חלק
 outline = תוכן עניינים
 raw = קטע מקור
-page = עמוד
+page = עמוד

crates/typst-library/translations/hu.txt

@@ -4,5 +4,5 @@ equation = Egyenlet
 bibliography = Irodalomjegyzék
 heading = Fejezet
 outline = Tartalomjegyzék
-# raw = 
+# raw =
-page = oldal
+page = oldal

crates/typst-library/translations/is.txt

@@ -5,4 +5,4 @@ bibliography = Heimildaskrá
 heading = Kafli
 outline = Efnisyfirlit
 raw = Sýnishorn
-page = blaðsíða
+page = blaðsíða

crates/typst-library/translations/it.txt

@@ -5,4 +5,4 @@ bibliography = Bibliografia
 heading = Sezione
 outline = Indice
 raw = Codice
-page = pag.
+page = pag.

crates/typst-library/translations/ja.txt

@@ -5,4 +5,4 @@ bibliography = 参考文献
 heading = 節
 outline = 目次
 raw = リスト
-page = ページ
+page = ページ

crates/typst-library/translations/la.txt

@@ -5,4 +5,4 @@ bibliography = Conspectus librorum
 heading = Caput
 outline = Index capitum
 raw = Exemplum
-page = charta
+page = charta

crates/typst-library/translations/nb.txt

@@ -5,4 +5,4 @@ bibliography = Bibliografi
 heading = Kapittel
 outline = Innhold
 raw = Utskrift
-page = side
+page = side

crates/typst-library/translations/nl.txt

@@ -5,4 +5,4 @@ bibliography = Bibliografie
 heading = Hoofdstuk
 outline = Inhoudsopgave
 raw = Listing
-page = pagina
+page = pagina

crates/typst-library/translations/nn.txt

@@ -5,4 +5,4 @@ bibliography = Bibliografi
 heading = Kapittel
 outline = Innhald
 raw = Utskrift
-page = side
+page = side

crates/typst-library/translations/pl.txt

@@ -5,4 +5,4 @@ bibliography = Bibliografia
 heading = Sekcja
 outline = Spis treści
 raw = Program
-page = strona
+page = strona

crates/typst-library/translations/pt-PT.txt

@@ -1,8 +1,8 @@
-# figure = 
+# figure =
-# table = 
+# table =
-# equation = 
+# equation =
-# bibliography = 
+# bibliography =
 heading = Secção
 outline = Índice
-# raw = 
+# raw =
-page = página
+page = página

crates/typst-library/translations/pt.txt

@@ -5,4 +5,4 @@ bibliography = Bibliografia
 heading = Seção
 outline = Sumário
 raw = Listagem
-page = página
+page = página

crates/typst-library/translations/ro.txt

@@ -6,4 +6,4 @@ heading = Secțiunea
 outline = Cuprins
 # may be wrong
 raw = Listă
-page = pagina
+page = pagina

crates/typst-library/translations/ru.txt

@@ -5,4 +5,4 @@ bibliography = Библиография
 heading = Раздел
 outline = Содержание
 raw = Листинг
-page = с.
+page = с.

crates/typst-library/translations/sl.txt

@@ -5,4 +5,4 @@ bibliography = Literatura
 heading = Poglavje
 outline = Kazalo
 raw = Program
-page = stran
+page = stran

crates/typst-library/translations/sq.txt

@@ -5,4 +5,4 @@ bibliography = Bibliografi
 heading = Kapitull
 outline = Përmbajtja
 raw = List
-page = faqe
+page = faqe

crates/typst-library/translations/sr.txt

@@ -5,4 +5,4 @@ bibliography = Литература
 heading = Поглавље
 outline = Садржај
 raw = Програм
-page = страна
+page = страна

crates/typst-library/translations/sv.txt

@@ -5,4 +5,4 @@ bibliography = Bibliografi
 heading = Kapitel
 outline = Innehåll
 raw = Listing
-page = sida
+page = sida

crates/typst-library/translations/tl.txt

@@ -5,4 +5,4 @@ bibliography = Bibliograpiya
 heading = Seksyon
 outline = Talaan ng mga Nilalaman
 raw = Listahan
-# page = 
+# page =

crates/typst-library/translations/tr.txt

@@ -5,4 +5,4 @@ bibliography = Kaynakça
 heading = Bölüm
 outline = İçindekiler
 raw = Liste
-page = sayfa
+page = sayfa

crates/typst-library/translations/uk.txt

@@ -5,4 +5,4 @@ bibliography = Бібліографія
 heading = Розділ
 outline = Зміст
 raw = Лістинг
-page = c.
+page = c.

crates/typst-library/translations/vi.txt

@@ -6,4 +6,4 @@ heading = Phần
 outline = Mục lục
 # may be wrong
 raw = Chương trình
-page = trang
+page = trang

crates/typst-library/translations/zh-TW.txt

@@ -1,8 +1,8 @@
 figure = 圖
-# table = 
+# table =
 equation = 式
 bibliography = 書目
 heading = 小節
 outline = 目錄
 raw = 程式
-# page = 
+# page =

crates/typst-library/translations/zh.txt

@@ -5,4 +5,4 @@ bibliography = 参考文献
 heading = 小节
 outline = 目录
 raw = 代码
-# page = 
+# page =

crates/typst-syntax/src/lexer.rs

@@ -4,7 +4,7 @@ use unicode_script::{Script, UnicodeScript};
 use unicode_segmentation::UnicodeSegmentation;
 use unscanny::Scanner;
 
-use crate::{SyntaxError, SyntaxKind, SyntaxNode};
+use crate::{SyntaxError, SyntaxKind, SyntaxMode, SyntaxNode};
 
 /// An iterator over a source code string which returns tokens.
 #[derive(Clone)]
@@ -13,28 +13,17 @@ pub(super) struct Lexer<'s> {
     s: Scanner<'s>,
     /// The mode the lexer is in. This determines which kinds of tokens it
     /// produces.
-    mode: LexMode,
+    mode: SyntaxMode,
     /// Whether the last token contained a newline.
     newline: bool,
     /// An error for the last token.
     error: Option<SyntaxError>,
 }
 
-/// What kind of tokens to emit.
-#[derive(Debug, Copy, Clone, Eq, PartialEq)]
-pub(super) enum LexMode {
-    /// Text and markup.
-    Markup,
-    /// Math atoms, operators, etc.
-    Math,
-    /// Keywords, literals and operators.
-    Code,
-}
-
 impl<'s> Lexer<'s> {
     /// Create a new lexer with the given mode and a prefix to offset column
     /// calculations.
-    pub fn new(text: &'s str, mode: LexMode) -> Self {
+    pub fn new(text: &'s str, mode: SyntaxMode) -> Self {
         Self {
             s: Scanner::new(text),
             mode,
@@ -44,12 +33,12 @@ impl<'s> Lexer<'s> {
     }
 
     /// Get the current lexing mode.
-    pub fn mode(&self) -> LexMode {
+    pub fn mode(&self) -> SyntaxMode {
         self.mode
     }
 
     /// Change the lexing mode.
-    pub fn set_mode(&mut self, mode: LexMode) {
+    pub fn set_mode(&mut self, mode: SyntaxMode) {
         self.mode = mode;
     }
 
@@ -92,7 +81,7 @@ impl Lexer<'_> {
     }
 }
 
-/// Shared methods with all [`LexMode`].
+/// Shared methods with all [`SyntaxMode`].
 impl Lexer<'_> {
     /// Return the next token in our text. Returns both the [`SyntaxNode`]
     /// and the raw [`SyntaxKind`] to make it more ergonomic to check the kind
@@ -114,14 +103,14 @@ impl Lexer<'_> {
                 );
                 kind
             }
-            Some('`') if self.mode != LexMode::Math => return self.raw(),
+            Some('`') if self.mode != SyntaxMode::Math => return self.raw(),
             Some(c) => match self.mode {
-                LexMode::Markup => self.markup(start, c),
+                SyntaxMode::Markup => self.markup(start, c),
-                LexMode::Math => match self.math(start, c) {
+                SyntaxMode::Math => match self.math(start, c) {
                     (kind, None) => kind,
                     (kind, Some(node)) => return (kind, node),
                 },
-                LexMode::Code => self.code(start, c),
+                SyntaxMode::Code => self.code(start, c),
             },
 
             None => SyntaxKind::End,
@@ -145,7 +134,7 @@ impl Lexer<'_> {
         };
 
         self.newline = newlines > 0;
-        if self.mode == LexMode::Markup && newlines >= 2 {
+        if self.mode == SyntaxMode::Markup && newlines >= 2 {
             SyntaxKind::Parbreak
         } else {
             SyntaxKind::Space
@@ -965,9 +954,9 @@ impl ScannerExt for Scanner<'_> {
 
 /// Whether a character will become a [`SyntaxKind::Space`] token.
 #[inline]
-fn is_space(character: char, mode: LexMode) -> bool {
+fn is_space(character: char, mode: SyntaxMode) -> bool {
     match mode {
-        LexMode::Markup => matches!(character, ' ' | '\t') || is_newline(character),
+        SyntaxMode::Markup => matches!(character, ' ' | '\t') || is_newline(character),
         _ => character.is_whitespace(),
     }
 }

crates/typst-syntax/src/lib.rs

@@ -30,5 +30,16 @@ pub use self::path::VirtualPath;
 pub use self::source::Source;
 pub use self::span::{Span, Spanned};
 
-use self::lexer::{LexMode, Lexer};
+use self::lexer::Lexer;
 use self::parser::{reparse_block, reparse_markup};
+
+/// The syntax mode of a portion of Typst code.
+#[derive(Debug, Copy, Clone, Eq, PartialEq, Hash)]
+pub enum SyntaxMode {
+    /// Text and markup, as in the top level.
+    Markup,
+    /// Math atoms, operators, etc., as in equations.
+    Math,
+    /// Keywords, literals and operators, as after hashes.
+    Code,
+}

crates/typst-syntax/src/parser.rs

@@ -7,12 +7,12 @@ use typst_utils::default_math_class;
 use unicode_math_class::MathClass;
 
 use crate::set::{syntax_set, SyntaxSet};
-use crate::{ast, set, LexMode, Lexer, SyntaxError, SyntaxKind, SyntaxNode};
+use crate::{ast, set, Lexer, SyntaxError, SyntaxKind, SyntaxMode, SyntaxNode};
 
 /// Parses a source file as top-level markup.
 pub fn parse(text: &str) -> SyntaxNode {
     let _scope = typst_timing::TimingScope::new("parse");
-    let mut p = Parser::new(text, 0, LexMode::Markup);
+    let mut p = Parser::new(text, 0, SyntaxMode::Markup);
     markup_exprs(&mut p, true, syntax_set!(End));
     p.finish_into(SyntaxKind::Markup)
 }
@@ -20,7 +20,7 @@ pub fn parse(text: &str) -> SyntaxNode {
 /// Parses top-level code.
 pub fn parse_code(text: &str) -> SyntaxNode {
     let _scope = typst_timing::TimingScope::new("parse code");
-    let mut p = Parser::new(text, 0, LexMode::Code);
+    let mut p = Parser::new(text, 0, SyntaxMode::Code);
     code_exprs(&mut p, syntax_set!(End));
     p.finish_into(SyntaxKind::Code)
 }
@@ -28,7 +28,7 @@ pub fn parse_code(text: &str) -> SyntaxNode {
 /// Parses top-level math.
 pub fn parse_math(text: &str) -> SyntaxNode {
     let _scope = typst_timing::TimingScope::new("parse math");
-    let mut p = Parser::new(text, 0, LexMode::Math);
+    let mut p = Parser::new(text, 0, SyntaxMode::Math);
     math_exprs(&mut p, syntax_set!(End));
     p.finish_into(SyntaxKind::Math)
 }
@@ -63,7 +63,7 @@ pub(super) fn reparse_markup(
     nesting: &mut usize,
     top_level: bool,
 ) -> Option<Vec<SyntaxNode>> {
-    let mut p = Parser::new(text, range.start, LexMode::Markup);
+    let mut p = Parser::new(text, range.start, SyntaxMode::Markup);
     *at_start |= p.had_newline();
     while !p.end() && p.current_start() < range.end {
         // If not top-level and at a new RightBracket, stop the reparse.
@@ -205,7 +205,7 @@ fn reference(p: &mut Parser) {
 /// Parses a mathematical equation: `$x$`, `$ x^2 $`.
 fn equation(p: &mut Parser) {
     let m = p.marker();
-    p.enter_modes(LexMode::Math, AtNewline::Continue, |p| {
+    p.enter_modes(SyntaxMode::Math, AtNewline::Continue, |p| {
         p.assert(SyntaxKind::Dollar);
         math(p, syntax_set!(Dollar, End));
         p.expect_closing_delimiter(m, SyntaxKind::Dollar);
@@ -615,7 +615,7 @@ fn code_exprs(p: &mut Parser, stop_set: SyntaxSet) {
 
 /// Parses an atomic code expression embedded in markup or math.
 fn embedded_code_expr(p: &mut Parser) {
-    p.enter_modes(LexMode::Code, AtNewline::Stop, |p| {
+    p.enter_modes(SyntaxMode::Code, AtNewline::Stop, |p| {
         p.assert(SyntaxKind::Hash);
         if p.had_trivia() || p.end() {
             p.expected("expression");
@@ -777,7 +777,7 @@ fn code_primary(p: &mut Parser, atomic: bool) {
 
 /// Reparses a full content or code block.
 pub(super) fn reparse_block(text: &str, range: Range<usize>) -> Option<SyntaxNode> {
-    let mut p = Parser::new(text, range.start, LexMode::Code);
+    let mut p = Parser::new(text, range.start, SyntaxMode::Code);
     assert!(p.at(SyntaxKind::LeftBracket) || p.at(SyntaxKind::LeftBrace));
     block(&mut p);
     (p.balanced && p.prev_end() == range.end)
@@ -796,7 +796,7 @@ fn block(p: &mut Parser) {
 /// Parses a code block: `{ let x = 1; x + 2 }`.
 fn code_block(p: &mut Parser) {
     let m = p.marker();
-    p.enter_modes(LexMode::Code, AtNewline::Continue, |p| {
+    p.enter_modes(SyntaxMode::Code, AtNewline::Continue, |p| {
         p.assert(SyntaxKind::LeftBrace);
         code(p, syntax_set!(RightBrace, RightBracket, RightParen, End));
         p.expect_closing_delimiter(m, SyntaxKind::RightBrace);
@@ -807,7 +807,7 @@ fn code_block(p: &mut Parser) {
 /// Parses a content block: `[*Hi* there!]`.
 fn content_block(p: &mut Parser) {
     let m = p.marker();
-    p.enter_modes(LexMode::Markup, AtNewline::Continue, |p| {
+    p.enter_modes(SyntaxMode::Markup, AtNewline::Continue, |p| {
         p.assert(SyntaxKind::LeftBracket);
         markup(p, true, true, syntax_set!(RightBracket, End));
         p.expect_closing_delimiter(m, SyntaxKind::RightBracket);
@@ -1516,10 +1516,10 @@ fn pattern_leaf<'s>(
 /// ### Modes
 ///
 /// The parser manages the transitions between the three modes of Typst through
-/// [lexer modes](`LexMode`) and [newline modes](`AtNewline`).
+/// [syntax modes](`SyntaxMode`) and [newline modes](`AtNewline`).
 ///
-/// The lexer modes map to the three Typst modes and are stored in the lexer,
+/// The syntax modes map to the three Typst modes and are stored in the lexer,
-/// changing which`SyntaxKind`s it will generate.
+/// changing which `SyntaxKind`s it will generate.
 ///
 /// The newline mode is used to determine whether a newline should end the
 /// current expression. If so, the parser temporarily changes `token`'s kind to
@@ -1529,7 +1529,7 @@ struct Parser<'s> {
     /// The source text shared with the lexer.
     text: &'s str,
     /// A lexer over the source text with multiple modes. Defines the boundaries
-    /// of tokens and determines their [`SyntaxKind`]. Contains the [`LexMode`]
+    /// of tokens and determines their [`SyntaxKind`]. Contains the [`SyntaxMode`]
     /// defining our current Typst mode.
     lexer: Lexer<'s>,
     /// The newline mode: whether to insert a temporary end at newlines.
@@ -1612,7 +1612,7 @@ impl AtNewline {
             AtNewline::RequireColumn(min_col) => {
                 // When the column is `None`, the newline doesn't start a
                 // column, and we continue parsing. This may happen on the
-                // boundary of lexer modes, since we only report a column in
+                // boundary of syntax modes, since we only report a column in
                 // Markup.
                 column.is_some_and(|column| column <= min_col)
             }
@@ -1643,8 +1643,8 @@ impl IndexMut<Marker> for Parser<'_> {
 
 /// Creating/Consuming the parser and getting info about the current token.
 impl<'s> Parser<'s> {
-    /// Create a new parser starting from the given text offset and lexer mode.
+    /// Create a new parser starting from the given text offset and syntax mode.
-    fn new(text: &'s str, offset: usize, mode: LexMode) -> Self {
+    fn new(text: &'s str, offset: usize, mode: SyntaxMode) -> Self {
         let mut lexer = Lexer::new(text, mode);
         lexer.jump(offset);
         let nl_mode = AtNewline::Continue;
@@ -1825,13 +1825,13 @@ impl<'s> Parser<'s> {
         self.nodes.insert(from, SyntaxNode::inner(kind, children));
     }
 
-    /// Parse within the [`LexMode`] for subsequent tokens (does not change the
+    /// Parse within the [`SyntaxMode`] for subsequent tokens (does not change the
     /// current token). This may re-lex the final token on exit.
     ///
     /// This function effectively repurposes the call stack as a stack of modes.
     fn enter_modes(
         &mut self,
-        mode: LexMode,
+        mode: SyntaxMode,
         stop: AtNewline,
         func: impl FnOnce(&mut Parser<'s>),
     ) {
@@ -1891,7 +1891,8 @@ impl<'s> Parser<'s> {
         }
 
         let newline = if had_newline {
-            let column = (lexer.mode() == LexMode::Markup).then(|| lexer.column(start));
+            let column =
+                (lexer.mode() == SyntaxMode::Markup).then(|| lexer.column(start));
             let newline = Newline { column, parbreak };
             if nl_mode.stop_at(newline, kind) {
                 // Insert a temporary `SyntaxKind::End` to halt the parser.
@@ -1938,7 +1939,7 @@ struct Checkpoint {
 #[derive(Clone)]
 struct PartialState {
     cursor: usize,
-    lex_mode: LexMode,
+    lex_mode: SyntaxMode,
     token: Token,
 }
 

crates/typst-utils/src/pico.rs

@@ -72,7 +72,7 @@ impl PicoStr {
     pub const fn constant(string: &'static str) -> PicoStr {
         match PicoStr::try_constant(string) {
             Ok(value) => value,
-            Err(err) => panic!("{}", err.message()),
+            Err(err) => failed_to_compile_time_intern(err, string),
         }
     }
 
@@ -190,15 +190,9 @@ mod bitcode {
     impl EncodingError {
         pub const fn message(&self) -> &'static str {
             match self {
-                Self::TooLong => {
+                Self::TooLong => "the maximum auto-internible string length is 12",
-                    "the maximum auto-internible string length is 12. \
-                     you can add an exception to typst-utils/src/pico.rs \
-                     to intern longer strings."
-                }
                 Self::BadChar => {
-                    "can only auto-intern the chars 'a'-'z', '1'-'4', and '-'. \
+                    "can only auto-intern the chars 'a'-'z', '1'-'4', and '-'"
-                     you can add an exception to typst-utils/src/pico.rs \
-                     to intern other strings."
                 }
             }
         }
@@ -356,6 +350,39 @@ impl Hash for ResolvedPicoStr {
     }
 }
 
+/// The error when a string could not be interned at compile time. Because the
+/// normal formatting machinery is not available at compile time, just producing
+/// the message is a bit involved ...
+#[track_caller]
+const fn failed_to_compile_time_intern(
+    error: bitcode::EncodingError,
+    string: &'static str,
+) -> ! {
+    const CAPACITY: usize = 512;
+    const fn push((buf, i): &mut ([u8; CAPACITY], usize), s: &str) {
+        let mut k = 0;
+        while k < s.len() && *i < buf.len() {
+            buf[*i] = s.as_bytes()[k];
+            k += 1;
+            *i += 1;
+        }
+    }
+
+    let mut dest = ([0; CAPACITY], 0);
+    push(&mut dest, "failed to compile-time intern string \"");
+    push(&mut dest, string);
+    push(&mut dest, "\". ");
+    push(&mut dest, error.message());
+    push(&mut dest, ". you can add an exception to ");
+    push(&mut dest, file!());
+    push(&mut dest, " to intern longer strings.");
+
+    let (slice, _) = dest.0.split_at(dest.1);
+    let Ok(message) = std::str::from_utf8(slice) else { panic!() };
+
+    panic!("{}", message);
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

tests/suite/pdf/embed.typ

@@ -28,3 +28,7 @@
   mime-type: "text/plain",
   description: "A test file",
 )
+
+--- pdf-embed-invalid-data ---
+// Error: 38-45 expected bytes, found string
+#pdf.embed("/assets/text/hello.txt", "hello")