Merge 476096c2db5f4d539a4cb0c4cb24e4e1a36a022e into b790c6d59ceaf7a809cc24b60c1f1509807470e2

Using "codepoint" is more accurate and lines up with what typst's standard library uses

Cargo.lock

@@ -413,7 +413,7 @@ dependencies = [
 [[package]]
 name = "codex"
 version = "0.1.1"
-source = "git+https://github.com/typst/codex?rev=9ac86f9#9ac86f96af5b89fce555e6bba8b6d1ac7b44ef00"
+source = "git+https://github.com/typst/codex?rev=775d828#775d82873c3f74ce95ec2621f8541de1b48778a7"
 
 [[package]]
 name = "color-print"

Cargo.toml

@@ -47,7 +47,7 @@ clap = { version = "4.4", features = ["derive", "env", "wrap_help"] }
 clap_complete = "4.2.1"
 clap_mangen = "0.2.10"
 codespan-reporting = "0.11"
-codex = { git = "https://github.com/typst/codex", rev = "9ac86f9" }
+codex = { git = "https://github.com/typst/codex", rev = "775d828" }
 color-print = "0.3.6"
 comemo = "0.4"
 csv = "1"

README.md

@@ -173,8 +173,11 @@ typst help
 typst help watch
 ```
 
-If you prefer an integrated IDE-like experience with autocompletion and instant
+If you prefer an integrated IDE-like experience with autocompletion and instant 
-preview, you can also check out [Typst's free web app][app].
+preview, you can also check out our [free web app][app]. Alternatively, there is 
+a community-created language server called 
+[Tinymist](https://myriad-dreamin.github.io/tinymist/) which is integrated into 
+various editor extensions.
 
 ## Community
 The main places where the community gathers are our [Forum][forum] and our

crates/typst-eval/src/markup.rs

@@ -123,7 +123,7 @@ impl Eval for ast::Escape<'_> {
     type Output = Value;
 
     fn eval(self, _: &mut Vm) -> SourceResult<Self::Output> {
-        Ok(Value::Symbol(Symbol::single(self.get())))
+        Ok(Value::Symbol(Symbol::runtime_char(self.get())))
     }
 }
 
@@ -131,7 +131,7 @@ impl Eval for ast::Shorthand<'_> {
     type Output = Value;
 
     fn eval(self, _: &mut Vm) -> SourceResult<Self::Output> {
-        Ok(Value::Symbol(Symbol::single(self.get())))
+        Ok(Value::Symbol(Symbol::runtime_char(self.get())))
     }
 }
 

crates/typst-eval/src/math.rs

@@ -49,7 +49,7 @@ impl Eval for ast::MathShorthand<'_> {
     type Output = Value;
 
     fn eval(self, _: &mut Vm) -> SourceResult<Self::Output> {
-        Ok(Value::Symbol(Symbol::single(self.get())))
+        Ok(Value::Symbol(Symbol::runtime_char(self.get())))
     }
 }
 

crates/typst-ide/src/complete.rs

@@ -98,7 +98,7 @@ pub enum CompletionKind {
     /// A font family.
     Font,
     /// A symbol.
-    Symbol(char),
+    Symbol(EcoString),
 }
 
 /// Complete in comments. Or rather, don't!
@@ -457,7 +457,7 @@ fn field_access_completions(
             for modifier in symbol.modifiers() {
                 if let Ok(modified) = symbol.clone().modified((), modifier) {
                     ctx.completions.push(Completion {
-                        kind: CompletionKind::Symbol(modified.get()),
+                        kind: CompletionKind::Symbol(modified.get().into()),
                         label: modifier.into(),
                         apply: None,
                         detail: None,
@@ -1392,7 +1392,7 @@ impl<'a> CompletionContext<'a> {
             kind: kind.unwrap_or_else(|| match value {
                 Value::Func(_) => CompletionKind::Func,
                 Value::Type(_) => CompletionKind::Type,
-                Value::Symbol(s) => CompletionKind::Symbol(s.get()),
+                Value::Symbol(s) => CompletionKind::Symbol(s.get().into()),
                 _ => CompletionKind::Constant,
             }),
             label,

crates/typst-layout/src/math/text.rs

@@ -129,12 +129,22 @@ pub fn layout_symbol(
     ctx: &mut MathContext,
     styles: StyleChain,
 ) -> SourceResult<()> {
+    assert!(
+        elem.text.len() <= 4 && elem.text.chars().count() == 1,
+        "TODO: layout multi-char symbol"
+    );
+    let elem_char = elem
+        .text
+        .chars()
+        .next()
+        .expect("TODO: should an empty symbol value forbidden?");
+
     // Switch dotless char to normal when we have the dtls OpenType feature.
     // This should happen before the main styling pass.
     let dtls = style_dtls();
-    let (unstyled_c, symbol_styles) = match try_dotless(elem.text) {
+    let (unstyled_c, symbol_styles) = match try_dotless(elem_char) {
         Some(c) if has_dtls_feat(ctx.font) => (c, styles.chain(&dtls)),
-        _ => (elem.text, styles),
+        _ => (elem_char, styles),
     };
 
     let variant = styles.get(EquationElem::variant);

crates/typst-library/src/foundations/symbol.rs

@@ -1,5 +1,5 @@
 use std::collections::{BTreeSet, HashMap};
-use std::fmt::{self, Debug, Display, Formatter, Write};
+use std::fmt::{self, Debug, Display, Formatter};
 use std::sync::Arc;
 
 use codex::ModifierSet;
@@ -52,7 +52,7 @@ pub struct Symbol(Repr);
 #[derive(Clone, Eq, PartialEq, Hash)]
 enum Repr {
     /// A native symbol that has no named variant.
-    Single(char),
+    Single(&'static str),
     /// A native symbol with multiple named variants.
     Complex(&'static [Variant<&'static str>]),
     /// A symbol with multiple named variants, where some modifiers may have
@@ -61,9 +61,9 @@ enum Repr {
     Modified(Arc<(List, ModifierSet<EcoString>)>),
 }
 
-/// A symbol variant, consisting of a set of modifiers, a character, and an
+/// A symbol variant, consisting of a set of modifiers, the variant's value, and an
 /// optional deprecation message.
-type Variant<S> = (ModifierSet<S>, char, Option<S>);
+type Variant<S> = (ModifierSet<S>, S, Option<S>);
 
 /// A collection of symbols.
 #[derive(Clone, Eq, PartialEq, Hash)]
@@ -73,9 +73,9 @@ enum List {
 }
 
 impl Symbol {
-    /// Create a new symbol from a single character.
+    /// Create a new symbol from a single value.
-    pub const fn single(c: char) -> Self {
+    pub const fn single(value: &'static str) -> Self {
-        Self(Repr::Single(c))
+        Self(Repr::Single(value))
     }
 
     /// Create a symbol with a static variant list.
@@ -85,6 +85,11 @@ impl Symbol {
         Self(Repr::Complex(list))
     }
 
+    /// Create a symbol from a runtime char.
+    pub fn runtime_char(c: char) -> Self {
+        Self::runtime(Box::new([(ModifierSet::default(), c.into(), None)]))
+    }
+
     /// Create a symbol with a runtime variant list.
     #[track_caller]
     pub fn runtime(list: Box<[Variant<EcoString>]>) -> Self {
@@ -92,15 +97,15 @@ impl Symbol {
         Self(Repr::Modified(Arc::new((List::Runtime(list), ModifierSet::default()))))
     }
 
-    /// Get the symbol's character.
+    /// Get the symbol's value.
-    pub fn get(&self) -> char {
+    pub fn get(&self) -> &str {
         match &self.0 {
-            Repr::Single(c) => *c,
+            Repr::Single(value) => value,
             Repr::Complex(_) => ModifierSet::<&'static str>::default()
-                .best_match_in(self.variants().map(|(m, c, _)| (m, c)))
+                .best_match_in(self.variants().map(|(m, v, _)| (m, v)))
                 .unwrap(),
             Repr::Modified(arc) => {
-                arc.1.best_match_in(self.variants().map(|(m, c, _)| (m, c))).unwrap()
+                arc.1.best_match_in(self.variants().map(|(m, v, _)| (m, v))).unwrap()
             }
         }
     }
@@ -108,27 +113,27 @@ impl Symbol {
     /// Try to get the function associated with the symbol, if any.
     pub fn func(&self) -> StrResult<Func> {
         match self.get() {
-            '⌈' => Ok(crate::math::ceil::func()),
+            "⌈" => Ok(crate::math::ceil::func()),
-            '⌊' => Ok(crate::math::floor::func()),
+            "⌊" => Ok(crate::math::floor::func()),
-            '–' => Ok(crate::math::accent::dash::func()),
+            "–" => Ok(crate::math::accent::dash::func()),
-            '⋅' | '\u{0307}' => Ok(crate::math::accent::dot::func()),
+            "⋅" | "\u{0307}" => Ok(crate::math::accent::dot::func()),
-            '¨' => Ok(crate::math::accent::dot_double::func()),
+            "¨" => Ok(crate::math::accent::dot_double::func()),
-            '\u{20db}' => Ok(crate::math::accent::dot_triple::func()),
+            "\u{20db}" => Ok(crate::math::accent::dot_triple::func()),
-            '\u{20dc}' => Ok(crate::math::accent::dot_quad::func()),
+            "\u{20dc}" => Ok(crate::math::accent::dot_quad::func()),
-            '∼' => Ok(crate::math::accent::tilde::func()),
+            "∼" => Ok(crate::math::accent::tilde::func()),
-            '´' => Ok(crate::math::accent::acute::func()),
+            "´" => Ok(crate::math::accent::acute::func()),
-            '˝' => Ok(crate::math::accent::acute_double::func()),
+            "˝" => Ok(crate::math::accent::acute_double::func()),
-            '˘' => Ok(crate::math::accent::breve::func()),
+            "˘" => Ok(crate::math::accent::breve::func()),
-            'ˇ' => Ok(crate::math::accent::caron::func()),
+            "ˇ" => Ok(crate::math::accent::caron::func()),
-            '^' => Ok(crate::math::accent::hat::func()),
+            "^" => Ok(crate::math::accent::hat::func()),
-            '`' => Ok(crate::math::accent::grave::func()),
+            "`" => Ok(crate::math::accent::grave::func()),
-            '¯' => Ok(crate::math::accent::macron::func()),
+            "¯" => Ok(crate::math::accent::macron::func()),
-            '○' => Ok(crate::math::accent::circle::func()),
+            "○" => Ok(crate::math::accent::circle::func()),
-            '→' => Ok(crate::math::accent::arrow::func()),
+            "→" => Ok(crate::math::accent::arrow::func()),
-            '←' => Ok(crate::math::accent::arrow_l::func()),
+            "←" => Ok(crate::math::accent::arrow_l::func()),
-            '↔' => Ok(crate::math::accent::arrow_l_r::func()),
+            "↔" => Ok(crate::math::accent::arrow_l_r::func()),
-            '⇀' => Ok(crate::math::accent::harpoon::func()),
+            "⇀" => Ok(crate::math::accent::harpoon::func()),
-            '↼' => Ok(crate::math::accent::harpoon_lt::func()),
+            "↼" => Ok(crate::math::accent::harpoon_lt::func()),
             _ => bail!("symbol {self} is not callable"),
         }
     }
@@ -163,7 +168,7 @@ impl Symbol {
     /// The characters that are covered by this symbol.
     pub fn variants(&self) -> impl Iterator<Item = Variant<&str>> {
         match &self.0 {
-            Repr::Single(c) => Variants::Single(Some(*c).into_iter()),
+            Repr::Single(value) => Variants::Single(std::iter::once(*value)),
             Repr::Complex(list) => Variants::Static(list.iter()),
             Repr::Modified(arc) => arc.0.variants(),
         }
@@ -279,14 +284,14 @@ impl Symbol {
 
 impl Display for Symbol {
     fn fmt(&self, f: &mut Formatter) -> fmt::Result {
-        f.write_char(self.get())
+        f.write_str(self.get())
     }
 }
 
 impl Debug for Repr {
     fn fmt(&self, f: &mut Formatter) -> fmt::Result {
         match self {
-            Self::Single(c) => Debug::fmt(c, f),
+            Self::Single(value) => Debug::fmt(value, f),
             Self::Complex(list) => list.fmt(f),
             Self::Modified(lists) => lists.fmt(f),
         }
@@ -305,7 +310,7 @@ impl Debug for List {
 impl crate::foundations::Repr for Symbol {
     fn repr(&self) -> EcoString {
         match &self.0 {
-            Repr::Single(c) => eco_format!("symbol(\"{}\")", *c),
+            Repr::Single(value) => eco_format!("symbol({})", value.repr()),
             Repr::Complex(variants) => {
                 eco_format!(
                     "symbol{}",
@@ -341,15 +346,15 @@ fn repr_variants<'a>(
                 // that contain all applied modifiers.
                 applied_modifiers.iter().all(|am| modifiers.contains(am))
             })
-            .map(|(modifiers, c, _)| {
+            .map(|(modifiers, value, _)| {
                 let trimmed_modifiers =
                     modifiers.into_iter().filter(|&m| !applied_modifiers.contains(m));
                 if trimmed_modifiers.clone().all(|m| m.is_empty()) {
-                    eco_format!("\"{c}\"")
+                    value.repr()
                 } else {
                     let trimmed_modifiers =
                         trimmed_modifiers.collect::<Vec<_>>().join(".");
-                    eco_format!("(\"{}\", \"{}\")", trimmed_modifiers, c)
+                    eco_format!("({}, {})", trimmed_modifiers.repr(), value.repr())
                 }
             })
             .collect::<Vec<_>>(),
@@ -362,7 +367,7 @@ impl Serialize for Symbol {
     where
         S: Serializer,
     {
-        serializer.serialize_char(self.get())
+        serializer.serialize_str(self.get())
     }
 }
 
@@ -377,11 +382,12 @@ impl List {
 }
 
 /// A value that can be cast to a symbol.
-pub struct SymbolVariant(EcoString, char);
+pub struct SymbolVariant(EcoString, EcoString);
 
 cast! {
     SymbolVariant,
-    c: char => Self(EcoString::new(), c),
+    c: char => Self(EcoString::new(), c.into()),
+    s: EcoString => Self(EcoString::new(), s),
     array: Array => {
         let mut iter = array.into_iter();
         match (iter.next(), iter.next(), iter.next()) {
@@ -393,7 +399,7 @@ cast! {
 
 /// Iterator over variants.
 enum Variants<'a> {
-    Single(std::option::IntoIter<char>),
+    Single(std::iter::Once<&'static str>),
     Static(std::slice::Iter<'static, Variant<&'static str>>),
     Runtime(std::slice::Iter<'a, Variant<EcoString>>),
 }
@@ -406,7 +412,7 @@ impl<'a> Iterator for Variants<'a> {
             Self::Single(iter) => Some((ModifierSet::default(), iter.next()?, None)),
             Self::Static(list) => list.next().copied(),
             Self::Runtime(list) => {
-                list.next().map(|(m, c, d)| (m.as_deref(), *c, d.as_deref()))
+                list.next().map(|(m, s, d)| (m.as_deref(), s.as_str(), d.as_deref()))
             }
         }
     }
@@ -415,21 +421,21 @@ impl<'a> Iterator for Variants<'a> {
 /// A single character.
 #[elem(Repr, PlainText)]
 pub struct SymbolElem {
-    /// The symbol's character.
+    /// The symbol's value.
     #[required]
-    pub text: char, // This is called `text` for consistency with `TextElem`.
+    pub text: EcoString, // This is called `text` for consistency with `TextElem`.
 }
 
 impl SymbolElem {
     /// Create a new packed symbol element.
-    pub fn packed(text: impl Into<char>) -> Content {
+    pub fn packed(text: impl Into<EcoString>) -> Content {
         Self::new(text.into()).pack()
     }
 }
 
 impl PlainText for Packed<SymbolElem> {
     fn plain_text(&self, text: &mut EcoString) {
-        text.push(self.text);
+        text.push_str(&self.text);
     }
 }
 

crates/typst-library/src/math/accent.rs

@@ -188,7 +188,7 @@ cast! {
     self => self.0.into_value(),
     v: char => Self::new(v),
     v: Content => match v.to_packed::<SymbolElem>() {
-        Some(elem) => Self::new(elem.text),
+        Some(elem) if elem.text.chars().count() == 1 => Self::new(elem.text.chars().next().unwrap()),
-        None => bail!("expected a symbol"),
+        _ => bail!("expected a single-codepoint symbol"),
     },
 }

crates/typst-library/src/math/matrix.rs

@@ -274,7 +274,7 @@ cast! {
     Delimiter,
     self => self.0.into_value(),
     _: NoneValue => Self::none(),
-    v: Symbol => Self::char(v.get())?,
+    v: Symbol => Self::char(v.get().parse::<char>().map_err(|_| "expected a single-codepoint symbol")?)?,
     v: char => Self::char(v)?,
 }
 

crates/typst-library/src/symbols.rs

@@ -39,7 +39,7 @@ impl From<codex::Module> for Scope {
 impl From<codex::Symbol> for Symbol {
     fn from(symbol: codex::Symbol) -> Self {
         match symbol {
-            codex::Symbol::Single(c) => Symbol::single(c),
+            codex::Symbol::Single(value) => Symbol::single(value),
             codex::Symbol::Multi(list) => Symbol::list(list),
         }
     }

crates/typst-library/src/visualize/color.rs

@@ -797,7 +797,9 @@ impl Color {
         components
     }
 
-    /// Returns the constructor function for this color's space:
+    /// Returns the constructor function for this color's space.
+    ///
+    /// Returns one of:
     /// - [`luma`]($color.luma)
     /// - [`oklab`]($color.oklab)
     /// - [`oklch`]($color.oklch)

crates/typst-realize/src/lib.rs

@@ -301,9 +301,7 @@ fn visit_kind_rules<'a>(
         // textual elements via `TEXTUAL` grouping. However, in math, this is
         // not desirable, so we just do it on a per-element basis.
         if let Some(elem) = content.to_packed::<SymbolElem>() {
-            if let Some(m) =
+            if let Some(m) = find_regex_match_in_str(elem.text.as_str(), styles) {
-                find_regex_match_in_str(elem.text.encode_utf8(&mut [0; 4]), styles)
-            {
                 visit_regex_match(s, &[(content, styles)], m)?;
                 return Ok(true);
             }
@@ -324,7 +322,7 @@ fn visit_kind_rules<'a>(
         // Symbols in non-math content transparently convert to `TextElem` so we
         // don't have to handle them in non-math layout.
         if let Some(elem) = content.to_packed::<SymbolElem>() {
-            let mut text = TextElem::packed(elem.text).spanned(elem.span());
+            let mut text = TextElem::packed(elem.text.clone()).spanned(elem.span());
             if let Some(label) = elem.label() {
                 text.set_label(label);
             }
@@ -1240,7 +1238,7 @@ fn visit_regex_match<'a>(
         let len = if let Some(elem) = content.to_packed::<TextElem>() {
             elem.text.len()
         } else if let Some(elem) = content.to_packed::<SymbolElem>() {
-            elem.text.len_utf8()
+            elem.text.len()
         } else {
             1 // The rest are Ascii, so just one byte.
         };

docs/src/lib.rs

@@ -242,7 +242,7 @@ fn category_page(resolver: &dyn Resolver, category: Category) -> PageModel {
             items.push(CategoryItem {
                 name: group.name.clone(),
                 route: subpage.route.clone(),
-                oneliner: oneliner(docs).into(),
+                oneliner: oneliner(docs),
                 code: true,
             });
             children.push(subpage);
@@ -296,7 +296,7 @@ fn category_page(resolver: &dyn Resolver, category: Category) -> PageModel {
                 items.push(CategoryItem {
                     name: name.into(),
                     route: subpage.route.clone(),
-                    oneliner: oneliner(func.docs().unwrap_or_default()).into(),
+                    oneliner: oneliner(func.docs().unwrap_or_default()),
                     code: true,
                 });
                 children.push(subpage);
@@ -306,7 +306,7 @@ fn category_page(resolver: &dyn Resolver, category: Category) -> PageModel {
                 items.push(CategoryItem {
                     name: ty.short_name().into(),
                     route: subpage.route.clone(),
-                    oneliner: oneliner(ty.docs()).into(),
+                    oneliner: oneliner(ty.docs()),
                     code: true,
                 });
                 children.push(subpage);
@@ -637,7 +637,7 @@ fn group_page(
     let item = CategoryItem {
         name: group.name.clone(),
         route: model.route.clone(),
-        oneliner: oneliner(&group.details).into(),
+        oneliner: oneliner(&group.details),
         code: false,
     };
 
@@ -718,9 +718,13 @@ fn symbols_model(resolver: &dyn Resolver, group: &GroupData) -> SymbolsModel {
             }
         };
 
-        for (variant, c, deprecation) in symbol.variants() {
+        for (variant, value, deprecation) in symbol.variants() {
+            let value_char = value.parse::<char>().ok();
+
             let shorthand = |list: &[(&'static str, char)]| {
-                list.iter().copied().find(|&(_, x)| x == c).map(|(s, _)| s)
+                value_char.and_then(|c| {
+                    list.iter().copied().find(|&(_, x)| x == c).map(|(s, _)| s)
+                })
             };
 
             let name = complete(variant);
@@ -729,9 +733,12 @@ fn symbols_model(resolver: &dyn Resolver, group: &GroupData) -> SymbolsModel {
                 name,
                 markup_shorthand: shorthand(typst::syntax::ast::Shorthand::LIST),
                 math_shorthand: shorthand(typst::syntax::ast::MathShorthand::LIST),
-                math_class: typst_utils::default_math_class(c).map(math_class_name),
+                math_class: value_char.and_then(|c| {
-                codepoint: c as _,
+                    typst_utils::default_math_class(c).map(math_class_name)
-                accent: typst::math::Accent::combine(c).is_some(),
+                }),
+                value: value.into(),
+                accent: value_char
+                    .is_some_and(|c| typst::math::Accent::combine(c).is_some()),
                 alternates: symbol
                     .variants()
                     .filter(|(other, _, _)| other != &variant)
@@ -772,8 +779,24 @@ pub fn urlify(title: &str) -> EcoString {
 }
 
 /// Extract the first line of documentation.
-fn oneliner(docs: &str) -> &str {
+fn oneliner(docs: &str) -> EcoString {
-    docs.lines().next().unwrap_or_default()
+    let paragraph = docs.split("\n\n").next().unwrap_or_default();
+    let mut depth = 0;
+    let mut period = false;
+    let mut end = paragraph.len();
+    for (i, c) in paragraph.char_indices() {
+        match c {
+            '(' | '[' | '{' => depth += 1,
+            ')' | ']' | '}' => depth -= 1,
+            '.' if depth == 0 => period = true,
+            c if period && c.is_whitespace() && !docs[..i].ends_with("e.g.") => {
+                end = i;
+                break;
+            }
+            _ => period = false,
+        }
+    }
+    EcoString::from(&docs[..end]).replace("\r\n", " ").replace("\n", " ")
 }
 
 /// The order of types in the documentation.

docs/src/model.rs

@@ -86,7 +86,7 @@ pub struct FuncModel {
     pub name: EcoString,
     pub title: &'static str,
     pub keywords: &'static [&'static str],
-    pub oneliner: &'static str,
+    pub oneliner: EcoString,
     pub element: bool,
     pub contextual: bool,
     pub deprecation: Option<&'static str>,
@@ -139,7 +139,7 @@ pub struct TypeModel {
     pub name: &'static str,
     pub title: &'static str,
     pub keywords: &'static [&'static str],
-    pub oneliner: &'static str,
+    pub oneliner: EcoString,
     pub details: Html,
     pub constructor: Option<FuncModel>,
     pub scope: Vec<FuncModel>,
@@ -159,7 +159,7 @@ pub struct SymbolsModel {
 #[serde(rename_all = "camelCase")]
 pub struct SymbolModel {
     pub name: EcoString,
-    pub codepoint: u32,
+    pub value: EcoString,
     pub accent: bool,
     pub alternates: Vec<EcoString>,
     pub markup_shorthand: Option<&'static str>,

flake.nix

@@ -127,6 +127,10 @@
           checks = self'.checks;
           inputsFrom = [ typst ];
 
+          buildInputs = with pkgs; [
+            rust-analyzer
+          ];
+
           packages = [
             # A script for quickly running tests.
             # See https://github.com/typst/typst/blob/main/tests/README.md#making-an-alias